puppy-tracker/server/pedigree.go

package main

// Pedigree lookup: resolve a dog by chip / registration number / name against
// SKK (Svenska Kennelklubben) HUNDDATA, then crawl its ancestry and expose it as
// an ahnentafel-positioned tree. SKK has no public API, so this scrapes the
// interactive ASP.NET WebForms app the same way a browser drives it:
//
//  1. Resolve  — POST Hund_sok.aspx/HundData (a JSON page-method) → hundid.
//  2. Fetch    — GET Hund_Stamtavla.aspx?hundid=X, then POST ddlGenerationer=7
//                to render 7 generations in one page; parse its rowspan grid.
//  3. Deepen   — each generation-7 leaf links via __doPostBack; POST that link
//                and read the ancestor's hundid out of the response __VIEWSTATE,
//                then recurse. BFS terminates when the ancestry runs out.
//
// A deep crawl is dozens of sequential requests, so a lookup returns the first
// 7 generations immediately and keeps crawling in the background; the finished
// tree is cached per hundid (pedigrees don't change) so a dog is crawled once.

import (
	"bytes"
	"context"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/html"
)

const (
	skkBase   = "https://hundar.skk.se/hunddata/"
	skkUA     = "Mozilla/5.0 (X11; Linux x86_64) puppy-tracker pedigree lookup"
	skkDelay  = 250 * time.Millisecond // politeness between upstream requests
	crawlGens = 7                      // generations SKK renders per page

	maxCrawlPages    = 400 // hard caps so a crawl can never run away
	maxCrawlRequests = 3000
	crawlDeadline    = 5 * time.Minute
	maxActiveJobs    = 4 // concurrent background crawls, total
	firstPageWait    = 20 * time.Second
)

// pedNode is one dog at an ahnentafel position (1 = subject, sire = 2n, dam = 2n+1).
type pedNode struct {
	Reg    string `json:"reg,omitempty"`
	Name   string `json:"name,omitempty"`
	Titles string `json:"titles,omitempty"`
	Hundid string `json:"hundid,omitempty"`
}

// pedSubject is the looked-up dog's headline info, from the resolver row.
type pedSubject struct {
	Hundid string `json:"hundid"`
	Reg    string `json:"reg"`
	Name   string `json:"name"`
	Breed  string `json:"breed"`
	Chip   string `json:"chip"`
	Sex    string `json:"sex,omitempty"`
}

// skkClient is a single browser-like session against SKK (cookie jar + UA).
type skkClient struct {
	hc *http.Client
}

func newSKKClient() (*skkClient, error) {
	jar, err := cookiejar.New(nil)
	if err != nil {
		return nil, err
	}
	return &skkClient{hc: &http.Client{Jar: jar, Timeout: 30 * time.Second}}, nil
}

func (c *skkClient) do(req *http.Request) (*http.Response, error) {
	req.Header.Set("User-Agent", skkUA)
	return c.hc.Do(req)
}

// warm establishes an ASP.NET session (SessionId + anti-XSRF cookies) that the
// resolver and pedigree pages both require.
func (c *skkClient) warm(ctx context.Context) error {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, skkBase+"Hund_sok.aspx", nil)
	if err != nil {
		return err
	}
	resp, err := c.do(req)
	if err != nil {
		return err
	}
	io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20))
	resp.Body.Close()
	return nil
}

// hundDataRow mirrors the fields the resolver page-method returns.
type hundDataRow struct {
	Hundid    string `json:"hundid"`
	Regnr     string `json:"Regnr"`
	Hundnamn  string `json:"hundnamn"`
	Chipnr    string `json:"chipnr"`
	Rastext   string `json:"rastext"`
	Kon       string `json:"Kon"`
	IDnummer  string `json:"IDnummer"`
	Antal     string `json:"Antal"`
	IsError   bool   `json:"IsError"`
	ErrorText string `json:"ErrorText"`
}

var digitsRE = regexp.MustCompile(`^\d+$`)

// resolve turns a user query (chip number, registration number, or name) into
// matching dogs. The field is chosen by shape: a long all-digit string is a
// chip; anything with a letter or slash is a registration number; otherwise a
// name search (which may return several rows to disambiguate).
func (c *skkClient) resolve(ctx context.Context, q string) ([]hundDataRow, error) {
	body := map[string]string{
		"txtRegnr": "", "txtIDnummer": "", "txtChipnr": "",
		"txtHundnamn": "", "ddlRasIn": "", "ddlKon": "", "txtLicensnr": "",
	}
	switch {
	case digitsRE.MatchString(q) && len(q) >= 10:
		body["txtChipnr"] = q
	case strings.ContainsAny(q, "/") || strings.IndexFunc(q, isLetter) >= 0:
		body["txtRegnr"] = q
	default:
		body["txtHundnamn"] = q
	}
	buf, _ := json.Marshal(body)
	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
		skkBase+"Hund_sok.aspx/HundData", bytes.NewReader(buf))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/json;charset=utf-8")
	req.Header.Set("Referer", skkBase+"Hund_sok.aspx")
	resp, err := c.do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	raw, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("resolver HTTP %d", resp.StatusCode)
	}
	var wrap struct {
		D []hundDataRow `json:"d"`
	}
	if err := json.Unmarshal(raw, &wrap); err != nil {
		return nil, fmt.Errorf("resolver response: %w", err)
	}
	// SKK signals "no matches" (and other soft failures like a query needing more
	// input) via a single IsError row rather than an HTTP error. Treat it as an
	// empty result so the caller reports a clean "not found" instead of a 502.
	if len(wrap.D) == 1 && wrap.D[0].IsError {
		return nil, nil
	}
	return wrap.D, nil
}

func isLetter(r rune) bool {
	return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')
}

// fetchPage GETs a dog's pedigree then POSTs ddlGenerationer=7 (with titles on)
// to render 7 generations, returning that page's HTML. The returned HTML is used
// for both grid parsing and the __doPostBack calls that resolve its leaf dogs,
// so its hidden fields (viewstate / event validation) match its ctl ids.
func (c *skkClient) fetchPage(ctx context.Context, hundid string) (string, string, error) {
	pageURL := skkBase + "Hund_Stamtavla.aspx?hundid=" + url.QueryEscape(hundid)
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
	if err != nil {
		return "", pageURL, err
	}
	req.Header.Set("Referer", skkBase+"Hund_sok.aspx")
	resp, err := c.do(req)
	if err != nil {
		return "", pageURL, err
	}
	raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
	resp.Body.Close()
	if err != nil {
		return "", pageURL, err
	}
	first := string(raw)

	form := hiddenFields(first)
	form.Set("ctl00$bodyContent$ddlGenerationer", strconv.Itoa(crawlGens))
	form.Set("ctl00$bodyContent$ddlTitlar", "J")
	form.Set("__EVENTTARGET", "ctl00$bodyContent$ddlGenerationer")
	form.Set("__EVENTARGUMENT", "")
	html7, err := c.postForm(ctx, pageURL, form)
	if err != nil {
		return "", pageURL, err
	}
	return html7, pageURL, nil
}

func (c *skkClient) postForm(ctx context.Context, pageURL string, form url.Values) (string, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, pageURL,
		strings.NewReader(form.Encode()))
	if err != nil {
		return "", err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("Referer", pageURL)
	resp, err := c.do(req)
	if err != nil {
		return "", err
	}
	raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
	resp.Body.Close()
	if err != nil {
		return "", err
	}
	return string(raw), nil
}

var (
	viewstateRE = regexp.MustCompile(`name="__VIEWSTATE" id="__VIEWSTATE" value="([^"]+)"`)
	// The clicked dog's internal hundid, encoded in the response viewstate as the
	// string "hundid", a type byte (\x05), a length byte, then ASCII digits.
	vsHundidRE = regexp.MustCompile(`(?s)hundid\x05.(\d+)`)
)

// postbackHundid clicks an ancestor's __doPostBack link on the given page and
// recovers that ancestor's hundid from the response viewstate. The rendered
// pedigree table never re-roots on such a click, but the viewstate carries the
// clicked dog's id — which is exactly the handle needed to fetch its own page.
func (c *skkClient) postbackHundid(ctx context.Context, pageHTML, ctlid, pageURL string) (string, error) {
	form := hiddenFields(pageHTML)
	form.Set("ctl00$bodyContent$ddlGenerationer", strconv.Itoa(crawlGens))
	form.Set("ctl00$bodyContent$ddlTitlar", "J")
	form.Set("__EVENTTARGET", ctlid)
	form.Set("__EVENTARGUMENT", "")
	resp, err := c.postForm(ctx, pageURL, form)
	if err != nil {
		return "", err
	}
	m := viewstateRE.FindStringSubmatch(resp)
	if m == nil {
		return "", nil
	}
	dec := decodeB64(m[1])
	mm := vsHundidRE.FindSubmatch(dec)
	if mm == nil {
		return "", nil
	}
	return string(mm[1]), nil
}

func decodeB64(s string) []byte {
	if m := len(s) % 4; m != 0 {
		s += strings.Repeat("=", 4-m)
	}
	b, err := base64.StdEncoding.DecodeString(s)
	if err != nil {
		return nil
	}
	return b
}

// hiddenFields collects every <input type=hidden> on a page into a form value
// set, so an ASP.NET postback can echo back __VIEWSTATE / __EVENTVALIDATION etc.
func hiddenFields(pageHTML string) url.Values {
	vals := url.Values{}
	node, err := html.Parse(strings.NewReader(pageHTML))
	if err != nil {
		return vals
	}
	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "input" {
			var typ, name, val string
			for _, a := range n.Attr {
				switch a.Key {
				case "type":
					typ = a.Val
				case "name":
					name = a.Val
				case "value":
					val = a.Val
				}
			}
			if typ == "hidden" && name != "" {
				vals.Set(name, val)
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(node)
	return vals
}

// gridCell is a parsed pedigree table cell.
type gridCell struct {
	reg, name, titles, ctlid string
	occupied                 bool
}

var doPostBackRE = regexp.MustCompile(`__doPostBack\('([^']+)'`)
var wsRE = regexp.MustCompile(`\s+`)

// parseGrid reconstructs the rowspan-based pedigree table into columns. Column c
// holds 2^c cells top-to-bottom; a cell's index within its column is its
// ahnentafel offset. Returns column index -> ordered cells.
func parseGrid(pageHTML string) map[int][]gridCell {
	node, err := html.Parse(strings.NewReader(pageHTML))
	if err != nil {
		return nil
	}
	tbl := findByID(node, "bodyContent_tblStamtavla")
	if tbl == nil {
		return nil
	}
	occ := map[[2]int]bool{}
	type placed struct {
		r, c int
		cell gridCell
	}
	var placedCells []placed
	r := 0
	for _, tr := range descendants(tbl, "tr") {
		c := 0
		for _, td := range directChildElements(tr, "td") {
			for occ[[2]int{r, c}] {
				c++
			}
			rs := 1
			if v := attr(td, "rowspan"); v != "" {
				if n, err := strconv.Atoi(v); err == nil && n > 0 {
					rs = n
				}
			}
			for dr := 0; dr < rs; dr++ {
				occ[[2]int{r + dr, c}] = true
			}
			placedCells = append(placedCells, placed{r, c, parseCell(td)})
			c++
		}
		r++
	}
	byCol := map[int][]placed{}
	for _, p := range placedCells {
		byCol[p.c] = append(byCol[p.c], p)
	}
	out := map[int][]gridCell{}
	for col, lst := range byCol {
		sort.SliceStable(lst, func(i, j int) bool { return lst[i].r < lst[j].r })
		cells := make([]gridCell, len(lst))
		for i, p := range lst {
			cells[i] = p.cell
		}
		out[col] = cells
	}
	return out
}

// parseCell extracts reg / name / titles / ctlid from one <td>, mirroring how
// SKK marks them up: a <a __doPostBack> holds the reg number (subject cells use
// a bold <span> instead), a <font> holds championship titles, and the last plain
// <span> holds the dog's name. "Uppgift saknas" placeholders are left unoccupied.
func parseCell(td *html.Node) gridCell {
	var cell gridCell
	if a := findElement(td, "a"); a != nil {
		if href := attr(a, "href"); strings.Contains(href, "__doPostBack") {
			cell.reg = normalizeText(text(a))
			if m := doPostBackRE.FindStringSubmatch(html.UnescapeString(href)); m != nil {
				cell.ctlid = m[1]
			}
		}
	}
	if f := findElement(td, "font"); f != nil {
		cell.titles = normalizeText(text(f))
	}
	// Name: the last <span> whose text isn't the titles string. Subject cells put
	// the reg in a leading bold span; if we found no link reg, adopt it.
	for _, sp := range descendants(td, "span") {
		t := normalizeText(text(sp))
		if t == "" || t == cell.titles {
			continue
		}
		cell.name = t
	}
	if cell.reg == "" {
		if b := findBoldSpan(td); b != "" {
			cell.reg = b
			if cell.name == b {
				cell.name = ""
			}
		}
	}
	if strings.EqualFold(cell.name, "Uppgift saknas") {
		cell.name = ""
	}
	cell.occupied = cell.reg != "" || cell.name != ""
	return cell
}

// crawlProgress is the mutable snapshot a running crawl publishes.
type crawlProgress struct {
	Pages    int
	Distinct int
	MaxGen   int
	Nodes    map[string]pedNode
}

// crawl performs the breadth-first ancestry walk from a subject hundid, calling
// report after each page with a fresh snapshot. It places every dog at its global
// ahnentafel position and resolves each generation-7 leaf to a hundid to recurse.
func (c *skkClient) crawl(ctx context.Context, hundid string, report func(crawlProgress)) (map[string]pedNode, error) {
	tree := map[string]pedNode{}
	edges := map[string]string{} // subjectHundid|ctlid -> ancestor hundid
	done := map[string]bool{}
	type qitem struct {
		hundid  string
		basePos uint64
	}
	queue := []qitem{{hundid, 1}}
	pages, requests, maxGen := 0, 0, 0

	snapshot := func() crawlProgress {
		nodes := make(map[string]pedNode, len(tree))
		for k, v := range tree {
			nodes[k] = v
		}
		return crawlProgress{Pages: pages, Distinct: countDistinct(tree), MaxGen: maxGen, Nodes: nodes}
	}

	for len(queue) > 0 {
		if err := ctx.Err(); err != nil {
			return tree, err
		}
		if pages >= maxCrawlPages || requests >= maxCrawlRequests {
			log.Printf("pedigree crawl %s: hit cap (pages=%d requests=%d)", hundid, pages, requests)
			break
		}
		item := queue[0]
		queue = queue[1:]
		if done[item.hundid] {
			continue
		}
		done[item.hundid] = true

		time.Sleep(skkDelay)
		pageHTML, pageURL, err := c.fetchPage(ctx, item.hundid)
		requests += 2
		if err != nil {
			if pages == 0 {
				return tree, err // couldn't even fetch the subject
			}
			log.Printf("pedigree crawl %s: fetch %s failed: %v", hundid, item.hundid, err)
			continue
		}
		pages++
		grid := parseGrid(pageHTML)

		var frontier []struct {
			gpos  uint64
			ctlid string
		}
		for col := 0; col <= crawlGens-1; col++ {
			cells, ok := grid[col]
			if !ok {
				continue
			}
			for pos, cell := range cells {
				if !cell.occupied {
					continue
				}
				gpos := item.basePos*(1<<uint(col)) + uint64(pos)
				key := strconv.FormatUint(gpos, 10)
				node := tree[key]
				node.Reg, node.Name, node.Titles = cell.reg, cell.name, cell.titles
				if col == 0 {
					node.Hundid = item.hundid
				}
				tree[key] = node
				if g := bitsLen(gpos); g > maxGen {
					maxGen = g
				}
				if col == crawlGens-1 && cell.ctlid != "" {
					frontier = append(frontier, struct {
						gpos  uint64
						ctlid string
					}{gpos, cell.ctlid})
				}
			}
		}
		report(snapshot())

		for _, f := range frontier {
			if err := ctx.Err(); err != nil {
				return tree, err
			}
			if requests >= maxCrawlRequests {
				break
			}
			ekey := item.hundid + "|" + f.ctlid
			hid, ok := edges[ekey]
			if !ok {
				time.Sleep(skkDelay)
				hid, err = c.postbackHundid(ctx, pageHTML, f.ctlid, pageURL)
				requests++
				if err != nil {
					continue
				}
				edges[ekey] = hid
			}
			if hid != "" {
				key := strconv.FormatUint(f.gpos, 10)
				node := tree[key]
				node.Hundid = hid
				tree[key] = node
				queue = append(queue, qitem{hid, f.gpos})
			}
		}
	}
	report(snapshot())
	return tree, nil
}

func bitsLen(x uint64) int {
	n := 0
	for x > 0 {
		n++
		x >>= 1
	}
	return n
}

// ---- job manager ---------------------------------------------------------

type jobState string

const (
	jobRunning jobState = "running"
	jobDone    jobState = "done"
	jobError   jobState = "error"
)

type pedJob struct {
	id        string
	hundid    string
	subject   pedSubject
	firstPage chan struct{} // closed once the subject's own page is parsed

	mu       sync.Mutex
	state    jobState
	pages    int
	distinct int
	maxGen   int
	nodes    map[string]pedNode
	errMsg   string
}

func (j *pedJob) apply(p crawlProgress) {
	j.mu.Lock()
	j.pages, j.distinct, j.maxGen, j.nodes = p.Pages, p.Distinct, p.MaxGen, p.Nodes
	j.mu.Unlock()
}

type pedManager struct {
	db *sql.DB

	mu   sync.Mutex
	jobs map[string]*pedJob // keyed by hundid (coalesces duplicate lookups)
}

func newPedManager(db *sql.DB) *pedManager {
	return &pedManager{db: db, jobs: map[string]*pedJob{}}
}

func (m *pedManager) activeCount() int {
	n := 0
	for _, j := range m.jobs {
		j.mu.Lock()
		if j.state == jobRunning {
			n++
		}
		j.mu.Unlock()
	}
	return n
}

// startOrAttach returns the running/finished job for a hundid, or starts a new
// background crawl. The boolean reports whether a fresh job was created.
func (m *pedManager) startOrAttach(client *skkClient, subject pedSubject) (*pedJob, bool, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if j, ok := m.jobs[subject.Hundid]; ok {
		return j, false, nil
	}
	if m.activeCount() >= maxActiveJobs {
		return nil, false, errors.New("busy: too many pedigree lookups in progress, try again shortly")
	}
	j := &pedJob{
		id:        subject.Hundid,
		hundid:    subject.Hundid,
		subject:   subject,
		firstPage: make(chan struct{}),
		state:     jobRunning,
		nodes:     map[string]pedNode{},
	}
	m.jobs[subject.Hundid] = j
	go m.run(client, j)
	return j, true, nil
}

func (m *pedManager) run(client *skkClient, j *pedJob) {
	ctx, cancel := context.WithTimeout(context.Background(), crawlDeadline)
	defer cancel()

	firstDone := false
	report := func(p crawlProgress) {
		j.apply(p)
		if !firstDone && p.Pages >= 1 {
			firstDone = true
			close(j.firstPage)
		}
	}
	nodes, err := client.crawl(ctx, j.hundid, report)
	if !firstDone {
		close(j.firstPage) // unblock waiters even if the very first fetch failed
	}
	j.mu.Lock()
	if err != nil && len(nodes) == 0 {
		j.state = jobError
		j.errMsg = err.Error()
	} else {
		j.state = jobDone
	}
	j.mu.Unlock()

	if len(nodes) > 0 {
		m.persist(j.subject, nodes)
	}
}

// snapshot copies a job's current public state under its lock.
func (j *pedJob) snapshot() (jobState, int, int, int, map[string]pedNode, string) {
	j.mu.Lock()
	defer j.mu.Unlock()
	nodes := make(map[string]pedNode, len(j.nodes))
	for k, v := range j.nodes {
		nodes[k] = v
	}
	return j.state, j.pages, j.distinct, j.maxGen, nodes, j.errMsg
}

// ---- persistent cache ----------------------------------------------------

func (m *pedManager) persist(subject pedSubject, nodes map[string]pedNode) {
	sj, _ := json.Marshal(subject)
	nj, _ := json.Marshal(nodes)
	_, err := m.db.Exec(`
		INSERT INTO pedigree_cache (hundid, subject, nodes, generations, fetched)
		VALUES (?, ?, ?, ?, ?)
		ON CONFLICT(hundid) DO UPDATE SET
			subject = excluded.subject, nodes = excluded.nodes,
			generations = excluded.generations, fetched = excluded.fetched`,
		subject.Hundid, string(sj), string(nj), maxGenerations(nodes), time.Now().UnixMilli())
	if err != nil {
		log.Printf("pedigree cache save %s: %v", subject.Hundid, err)
	}
}

// cached returns a stored tree for a hundid, if present.
func (m *pedManager) cached(hundid string) (pedSubject, map[string]pedNode, bool) {
	var sj, nj string
	err := m.db.QueryRow(
		`SELECT subject, nodes FROM pedigree_cache WHERE hundid = ?`, hundid,
	).Scan(&sj, &nj)
	if err != nil {
		if !errors.Is(err, sql.ErrNoRows) {
			log.Printf("pedigree cache get %s: %v", hundid, err)
		}
		return pedSubject{}, nil, false
	}
	var subject pedSubject
	var nodes map[string]pedNode
	json.Unmarshal([]byte(sj), &subject)
	json.Unmarshal([]byte(nj), &nodes)
	return subject, nodes, true
}

func maxGenerations(nodes map[string]pedNode) int {
	max := 0
	for k := range nodes {
		if p, err := strconv.ParseUint(k, 10, 64); err == nil {
			if g := bitsLen(p); g > max {
				max = g
			}
		}
	}
	return max
}

// ---- HTTP handlers -------------------------------------------------------

type pedLookupResponse struct {
	Status      string             `json:"status"` // done | running | choose
	JobID       string             `json:"jobId,omitempty"`
	Hundid      string             `json:"hundid,omitempty"`
	Subject     *pedSubject        `json:"subject,omitempty"`
	Generations int                `json:"generations,omitempty"`
	Nodes       map[string]pedNode `json:"nodes,omitempty"`
	Matches     []hundDataRow      `json:"matches,omitempty"`
}

func rowToSubject(r hundDataRow) pedSubject {
	return pedSubject{
		Hundid: r.Hundid,
		Reg:    strings.TrimSpace(r.Regnr),
		Name:   strings.TrimSpace(r.Hundnamn),
		Breed:  strings.TrimSpace(r.Rastext),
		Chip:   strings.TrimSpace(r.Chipnr),
		Sex:    strings.TrimSpace(r.Kon),
	}
}

// handleLookup resolves a query and returns a cached tree, a disambiguation list,
// or an immediate 7-generation tree with a background job crawling deeper.
func (m *pedManager) handleLookup(w http.ResponseWriter, r *http.Request) {
	var req struct {
		Q string `json:"q"`
	}
	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
		http.Error(w, "bad json: "+err.Error(), http.StatusBadRequest)
		return
	}
	q := strings.TrimSpace(req.Q)
	if q == "" {
		http.Error(w, "empty query", http.StatusBadRequest)
		return
	}

	client, err := newSKKClient()
	if err != nil {
		http.Error(w, "server error", http.StatusInternalServerError)
		return
	}
	ctx := r.Context()
	if err := client.warm(ctx); err != nil {
		http.Error(w, "upstream unavailable", http.StatusBadGateway)
		return
	}
	rows, err := client.resolve(ctx, q)
	if err != nil {
		http.Error(w, "lookup failed: "+err.Error(), http.StatusBadGateway)
		return
	}
	rows = withHundid(rows)
	switch {
	case len(rows) == 0:
		http.Error(w, "no dog found for "+q, http.StatusNotFound)
		return
	case len(rows) > 1:
		writeJSON(w, pedLookupResponse{Status: "choose", Matches: rows})
		return
	}
	subject := rowToSubject(rows[0])

	if subj, nodes, ok := m.cached(subject.Hundid); ok {
		writeJSON(w, pedLookupResponse{
			Status: "done", Hundid: subj.Hundid, Subject: &subj,
			Generations: maxGenerations(nodes), Nodes: nodes,
		})
		return
	}

	job, _, err := m.startOrAttach(client, subject)
	if err != nil {
		http.Error(w, err.Error(), http.StatusServiceUnavailable)
		return
	}
	select {
	case <-job.firstPage:
	case <-time.After(firstPageWait):
	case <-ctx.Done():
		return
	}
	state, _, _, gen, nodes, msg := job.snapshot()
	if state == jobError {
		http.Error(w, "pedigree fetch failed: "+msg, http.StatusBadGateway)
		return
	}
	status := "running"
	if state == jobDone {
		status = "done"
	}
	writeJSON(w, pedLookupResponse{
		Status: status, JobID: job.id, Hundid: subject.Hundid,
		Subject: &subject, Generations: gen, Nodes: nodes,
	})
}

type pedStatusResponse struct {
	Status      string             `json:"status"`
	Pages       int                `json:"pages"`
	Distinct    int                `json:"distinct"`
	Generations int                `json:"generations"`
	Nodes       map[string]pedNode `json:"nodes,omitempty"`
	Error       string             `json:"error,omitempty"`
}

// handleStatus returns a running crawl's current partial tree so the client can
// fill the view in progressively, and the final tree when it finishes.
func (m *pedManager) handleStatus(w http.ResponseWriter, r *http.Request) {
	id := r.URL.Query().Get("job")
	m.mu.Lock()
	job := m.jobs[id]
	m.mu.Unlock()
	if job == nil {
		// A finished job may have been evicted, but the cache still has the tree.
		if _, nodes, ok := m.cached(id); ok {
			writeJSON(w, pedStatusResponse{
				Status: "done", Generations: maxGenerations(nodes),
				Distinct: countDistinct(nodes), Nodes: nodes, Pages: 0,
			})
			return
		}
		http.Error(w, "unknown job", http.StatusNotFound)
		return
	}
	state, pages, distinct, gen, nodes, msg := job.snapshot()
	writeJSON(w, pedStatusResponse{
		Status: string(state), Pages: pages, Distinct: distinct,
		Generations: gen, Nodes: nodes, Error: msg,
	})
}

// countDistinct counts unique ancestors, keyed by registration number (falling
// back to name). Pedigree collapse means one dog can fill many positions, so
// this is smaller than the number of occupied positions.
func countDistinct(nodes map[string]pedNode) int {
	seen := map[string]bool{}
	for _, n := range nodes {
		k := n.Reg
		if k == "" {
			k = n.Name
		}
		if k != "" {
			seen[k] = true
		}
	}
	return len(seen)
}

func writeJSON(w http.ResponseWriter, v any) {
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Cache-Control", "no-store")
	_ = json.NewEncoder(w).Encode(v)
}

// withHundid drops resolver rows lacking a usable hundid (defensive).
func withHundid(rows []hundDataRow) []hundDataRow {
	out := rows[:0]
	for _, r := range rows {
		if strings.TrimSpace(r.Hundid) != "" {
			out = append(out, r)
		}
	}
	return out
}