Add pedigree lookup and ancestry tree page
New 🌳 Pedigree view: enter a dog's ISO chip or SKK registration number
and see its ancestry rendered as a tree. SKK has no public API, so the
server drives SKK Hunddata like a browser: it resolves the input to an
internal hundid via the Hund_sok.aspx/HundData page-method, renders 7
generations per pedigree page, parses the rowspan grid into ahnentafel
positions, and follows each generation's leaves deeper by reading their
hundid out of the __doPostBack response viewstate.
A lookup returns the first 7 generations immediately and crawls deeper in
the background; the client polls and fills the tree in as ancestors
arrive. Finished trees are cached per dog in a new pedigree_cache table
(pedigrees don't change), so a dog is crawled once and repeats are instant.
The endpoints sit behind auth like the rest of /api/*, and the crawl is
kept polite (warmed session, delay between requests, one coalesced job per
dog, hard caps).
This commit is contained in:
Alexander Heldt
@@ -0,0 +1,881 @@
|
||||
package main
|
||||
|
||||
// Pedigree lookup: resolve a dog by chip / registration number / name against
|
||||
// SKK (Svenska Kennelklubben) HUNDDATA, then crawl its ancestry and expose it as
|
||||
// an ahnentafel-positioned tree. SKK has no public API, so this scrapes the
|
||||
// interactive ASP.NET WebForms app the same way a browser drives it:
|
||||
//
|
||||
// 1. Resolve — POST Hund_sok.aspx/HundData (a JSON page-method) → hundid.
|
||||
// 2. Fetch — GET Hund_Stamtavla.aspx?hundid=X, then POST ddlGenerationer=7
|
||||
// to render 7 generations in one page; parse its rowspan grid.
|
||||
// 3. Deepen — each generation-7 leaf links via __doPostBack; POST that link
|
||||
// and read the ancestor's hundid out of the response __VIEWSTATE,
|
||||
// then recurse. BFS terminates when the ancestry runs out.
|
||||
//
|
||||
// A deep crawl is dozens of sequential requests, so a lookup returns the first
|
||||
// 7 generations immediately and keeps crawling in the background; the finished
|
||||
// tree is cached per hundid (pedigrees don't change) so a dog is crawled once.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
const (
|
||||
skkBase = "https://hundar.skk.se/hunddata/"
|
||||
skkUA = "Mozilla/5.0 (X11; Linux x86_64) puppy-tracker pedigree lookup"
|
||||
skkDelay = 250 * time.Millisecond // politeness between upstream requests
|
||||
crawlGens = 7 // generations SKK renders per page
|
||||
|
||||
maxCrawlPages = 400 // hard caps so a crawl can never run away
|
||||
maxCrawlRequests = 3000
|
||||
crawlDeadline = 5 * time.Minute
|
||||
maxActiveJobs = 4 // concurrent background crawls, total
|
||||
firstPageWait = 20 * time.Second
|
||||
)
|
||||
|
||||
// pedNode is one dog at an ahnentafel position (1 = subject, sire = 2n, dam = 2n+1).
|
||||
type pedNode struct {
|
||||
Reg string `json:"reg,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Titles string `json:"titles,omitempty"`
|
||||
Hundid string `json:"hundid,omitempty"`
|
||||
}
|
||||
|
||||
// pedSubject is the looked-up dog's headline info, from the resolver row.
|
||||
type pedSubject struct {
|
||||
Hundid string `json:"hundid"`
|
||||
Reg string `json:"reg"`
|
||||
Name string `json:"name"`
|
||||
Breed string `json:"breed"`
|
||||
Chip string `json:"chip"`
|
||||
Sex string `json:"sex,omitempty"`
|
||||
}
|
||||
|
||||
// skkClient is a single browser-like session against SKK (cookie jar + UA).
|
||||
type skkClient struct {
|
||||
hc *http.Client
|
||||
}
|
||||
|
||||
func newSKKClient() (*skkClient, error) {
|
||||
jar, err := cookiejar.New(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &skkClient{hc: &http.Client{Jar: jar, Timeout: 30 * time.Second}}, nil
|
||||
}
|
||||
|
||||
func (c *skkClient) do(req *http.Request) (*http.Response, error) {
|
||||
req.Header.Set("User-Agent", skkUA)
|
||||
return c.hc.Do(req)
|
||||
}
|
||||
|
||||
// warm establishes an ASP.NET session (SessionId + anti-XSRF cookies) that the
|
||||
// resolver and pedigree pages both require.
|
||||
func (c *skkClient) warm(ctx context.Context) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, skkBase+"Hund_sok.aspx", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := c.do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20))
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// hundDataRow mirrors the fields the resolver page-method returns.
|
||||
type hundDataRow struct {
|
||||
Hundid string `json:"hundid"`
|
||||
Regnr string `json:"Regnr"`
|
||||
Hundnamn string `json:"hundnamn"`
|
||||
Chipnr string `json:"chipnr"`
|
||||
Rastext string `json:"rastext"`
|
||||
Kon string `json:"Kon"`
|
||||
IDnummer string `json:"IDnummer"`
|
||||
Antal string `json:"Antal"`
|
||||
IsError bool `json:"IsError"`
|
||||
ErrorText string `json:"ErrorText"`
|
||||
}
|
||||
|
||||
var digitsRE = regexp.MustCompile(`^\d+$`)
|
||||
|
||||
// resolve turns a user query (chip number, registration number, or name) into
|
||||
// matching dogs. The field is chosen by shape: a long all-digit string is a
|
||||
// chip; anything with a letter or slash is a registration number; otherwise a
|
||||
// name search (which may return several rows to disambiguate).
|
||||
func (c *skkClient) resolve(ctx context.Context, q string) ([]hundDataRow, error) {
|
||||
body := map[string]string{
|
||||
"txtRegnr": "", "txtIDnummer": "", "txtChipnr": "",
|
||||
"txtHundnamn": "", "ddlRasIn": "", "ddlKon": "", "txtLicensnr": "",
|
||||
}
|
||||
switch {
|
||||
case digitsRE.MatchString(q) && len(q) >= 10:
|
||||
body["txtChipnr"] = q
|
||||
case strings.ContainsAny(q, "/") || strings.IndexFunc(q, isLetter) >= 0:
|
||||
body["txtRegnr"] = q
|
||||
default:
|
||||
body["txtHundnamn"] = q
|
||||
}
|
||||
buf, _ := json.Marshal(body)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
skkBase+"Hund_sok.aspx/HundData", bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json;charset=utf-8")
|
||||
req.Header.Set("Referer", skkBase+"Hund_sok.aspx")
|
||||
resp, err := c.do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("resolver HTTP %d", resp.StatusCode)
|
||||
}
|
||||
var wrap struct {
|
||||
D []hundDataRow `json:"d"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &wrap); err != nil {
|
||||
return nil, fmt.Errorf("resolver response: %w", err)
|
||||
}
|
||||
// SKK signals "no matches" (and other soft failures like a query needing more
|
||||
// input) via a single IsError row rather than an HTTP error. Treat it as an
|
||||
// empty result so the caller reports a clean "not found" instead of a 502.
|
||||
if len(wrap.D) == 1 && wrap.D[0].IsError {
|
||||
return nil, nil
|
||||
}
|
||||
return wrap.D, nil
|
||||
}
|
||||
|
||||
func isLetter(r rune) bool {
|
||||
return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')
|
||||
}
|
||||
|
||||
// fetchPage GETs a dog's pedigree then POSTs ddlGenerationer=7 (with titles on)
|
||||
// to render 7 generations, returning that page's HTML. The returned HTML is used
|
||||
// for both grid parsing and the __doPostBack calls that resolve its leaf dogs,
|
||||
// so its hidden fields (viewstate / event validation) match its ctl ids.
|
||||
func (c *skkClient) fetchPage(ctx context.Context, hundid string) (string, string, error) {
|
||||
pageURL := skkBase + "Hund_Stamtavla.aspx?hundid=" + url.QueryEscape(hundid)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
|
||||
if err != nil {
|
||||
return "", pageURL, err
|
||||
}
|
||||
req.Header.Set("Referer", skkBase+"Hund_sok.aspx")
|
||||
resp, err := c.do(req)
|
||||
if err != nil {
|
||||
return "", pageURL, err
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return "", pageURL, err
|
||||
}
|
||||
first := string(raw)
|
||||
|
||||
form := hiddenFields(first)
|
||||
form.Set("ctl00$bodyContent$ddlGenerationer", strconv.Itoa(crawlGens))
|
||||
form.Set("ctl00$bodyContent$ddlTitlar", "J")
|
||||
form.Set("__EVENTTARGET", "ctl00$bodyContent$ddlGenerationer")
|
||||
form.Set("__EVENTARGUMENT", "")
|
||||
html7, err := c.postForm(ctx, pageURL, form)
|
||||
if err != nil {
|
||||
return "", pageURL, err
|
||||
}
|
||||
return html7, pageURL, nil
|
||||
}
|
||||
|
||||
func (c *skkClient) postForm(ctx context.Context, pageURL string, form url.Values) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, pageURL,
|
||||
strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.Header.Set("Referer", pageURL)
|
||||
resp, err := c.do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(raw), nil
|
||||
}
|
||||
|
||||
var (
|
||||
viewstateRE = regexp.MustCompile(`name="__VIEWSTATE" id="__VIEWSTATE" value="([^"]+)"`)
|
||||
// The clicked dog's internal hundid, encoded in the response viewstate as the
|
||||
// string "hundid", a type byte (\x05), a length byte, then ASCII digits.
|
||||
vsHundidRE = regexp.MustCompile(`(?s)hundid\x05.(\d+)`)
|
||||
)
|
||||
|
||||
// postbackHundid clicks an ancestor's __doPostBack link on the given page and
|
||||
// recovers that ancestor's hundid from the response viewstate. The rendered
|
||||
// pedigree table never re-roots on such a click, but the viewstate carries the
|
||||
// clicked dog's id — which is exactly the handle needed to fetch its own page.
|
||||
func (c *skkClient) postbackHundid(ctx context.Context, pageHTML, ctlid, pageURL string) (string, error) {
|
||||
form := hiddenFields(pageHTML)
|
||||
form.Set("ctl00$bodyContent$ddlGenerationer", strconv.Itoa(crawlGens))
|
||||
form.Set("ctl00$bodyContent$ddlTitlar", "J")
|
||||
form.Set("__EVENTTARGET", ctlid)
|
||||
form.Set("__EVENTARGUMENT", "")
|
||||
resp, err := c.postForm(ctx, pageURL, form)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
m := viewstateRE.FindStringSubmatch(resp)
|
||||
if m == nil {
|
||||
return "", nil
|
||||
}
|
||||
dec := decodeB64(m[1])
|
||||
mm := vsHundidRE.FindSubmatch(dec)
|
||||
if mm == nil {
|
||||
return "", nil
|
||||
}
|
||||
return string(mm[1]), nil
|
||||
}
|
||||
|
||||
func decodeB64(s string) []byte {
|
||||
if m := len(s) % 4; m != 0 {
|
||||
s += strings.Repeat("=", 4-m)
|
||||
}
|
||||
b, err := base64.StdEncoding.DecodeString(s)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// hiddenFields collects every <input type=hidden> on a page into a form value
|
||||
// set, so an ASP.NET postback can echo back __VIEWSTATE / __EVENTVALIDATION etc.
|
||||
func hiddenFields(pageHTML string) url.Values {
|
||||
vals := url.Values{}
|
||||
node, err := html.Parse(strings.NewReader(pageHTML))
|
||||
if err != nil {
|
||||
return vals
|
||||
}
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode && n.Data == "input" {
|
||||
var typ, name, val string
|
||||
for _, a := range n.Attr {
|
||||
switch a.Key {
|
||||
case "type":
|
||||
typ = a.Val
|
||||
case "name":
|
||||
name = a.Val
|
||||
case "value":
|
||||
val = a.Val
|
||||
}
|
||||
}
|
||||
if typ == "hidden" && name != "" {
|
||||
vals.Set(name, val)
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(node)
|
||||
return vals
|
||||
}
|
||||
|
||||
// gridCell is a parsed pedigree table cell.
|
||||
type gridCell struct {
|
||||
reg, name, titles, ctlid string
|
||||
occupied bool
|
||||
}
|
||||
|
||||
var doPostBackRE = regexp.MustCompile(`__doPostBack\('([^']+)'`)
|
||||
var wsRE = regexp.MustCompile(`\s+`)
|
||||
|
||||
// parseGrid reconstructs the rowspan-based pedigree table into columns. Column c
|
||||
// holds 2^c cells top-to-bottom; a cell's index within its column is its
|
||||
// ahnentafel offset. Returns column index -> ordered cells.
|
||||
func parseGrid(pageHTML string) map[int][]gridCell {
|
||||
node, err := html.Parse(strings.NewReader(pageHTML))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
tbl := findByID(node, "bodyContent_tblStamtavla")
|
||||
if tbl == nil {
|
||||
return nil
|
||||
}
|
||||
occ := map[[2]int]bool{}
|
||||
type placed struct {
|
||||
r, c int
|
||||
cell gridCell
|
||||
}
|
||||
var placedCells []placed
|
||||
r := 0
|
||||
for _, tr := range descendants(tbl, "tr") {
|
||||
c := 0
|
||||
for _, td := range directChildElements(tr, "td") {
|
||||
for occ[[2]int{r, c}] {
|
||||
c++
|
||||
}
|
||||
rs := 1
|
||||
if v := attr(td, "rowspan"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
rs = n
|
||||
}
|
||||
}
|
||||
for dr := 0; dr < rs; dr++ {
|
||||
occ[[2]int{r + dr, c}] = true
|
||||
}
|
||||
placedCells = append(placedCells, placed{r, c, parseCell(td)})
|
||||
c++
|
||||
}
|
||||
r++
|
||||
}
|
||||
byCol := map[int][]placed{}
|
||||
for _, p := range placedCells {
|
||||
byCol[p.c] = append(byCol[p.c], p)
|
||||
}
|
||||
out := map[int][]gridCell{}
|
||||
for col, lst := range byCol {
|
||||
sort.SliceStable(lst, func(i, j int) bool { return lst[i].r < lst[j].r })
|
||||
cells := make([]gridCell, len(lst))
|
||||
for i, p := range lst {
|
||||
cells[i] = p.cell
|
||||
}
|
||||
out[col] = cells
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parseCell extracts reg / name / titles / ctlid from one <td>, mirroring how
|
||||
// SKK marks them up: a <a __doPostBack> holds the reg number (subject cells use
|
||||
// a bold <span> instead), a <font> holds championship titles, and the last plain
|
||||
// <span> holds the dog's name. "Uppgift saknas" placeholders are left unoccupied.
|
||||
func parseCell(td *html.Node) gridCell {
|
||||
var cell gridCell
|
||||
if a := findElement(td, "a"); a != nil {
|
||||
if href := attr(a, "href"); strings.Contains(href, "__doPostBack") {
|
||||
cell.reg = normalizeText(text(a))
|
||||
if m := doPostBackRE.FindStringSubmatch(html.UnescapeString(href)); m != nil {
|
||||
cell.ctlid = m[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
if f := findElement(td, "font"); f != nil {
|
||||
cell.titles = normalizeText(text(f))
|
||||
}
|
||||
// Name: the last <span> whose text isn't the titles string. Subject cells put
|
||||
// the reg in a leading bold span; if we found no link reg, adopt it.
|
||||
for _, sp := range descendants(td, "span") {
|
||||
t := normalizeText(text(sp))
|
||||
if t == "" || t == cell.titles {
|
||||
continue
|
||||
}
|
||||
cell.name = t
|
||||
}
|
||||
if cell.reg == "" {
|
||||
if b := findBoldSpan(td); b != "" {
|
||||
cell.reg = b
|
||||
if cell.name == b {
|
||||
cell.name = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.EqualFold(cell.name, "Uppgift saknas") {
|
||||
cell.name = ""
|
||||
}
|
||||
cell.occupied = cell.reg != "" || cell.name != ""
|
||||
return cell
|
||||
}
|
||||
|
||||
// crawlProgress is the mutable snapshot a running crawl publishes.
|
||||
type crawlProgress struct {
|
||||
Pages int
|
||||
Distinct int
|
||||
MaxGen int
|
||||
Nodes map[string]pedNode
|
||||
}
|
||||
|
||||
// crawl performs the breadth-first ancestry walk from a subject hundid, calling
|
||||
// report after each page with a fresh snapshot. It places every dog at its global
|
||||
// ahnentafel position and resolves each generation-7 leaf to a hundid to recurse.
|
||||
func (c *skkClient) crawl(ctx context.Context, hundid string, report func(crawlProgress)) (map[string]pedNode, error) {
|
||||
tree := map[string]pedNode{}
|
||||
edges := map[string]string{} // subjectHundid|ctlid -> ancestor hundid
|
||||
done := map[string]bool{}
|
||||
type qitem struct {
|
||||
hundid string
|
||||
basePos uint64
|
||||
}
|
||||
queue := []qitem{{hundid, 1}}
|
||||
pages, requests, maxGen := 0, 0, 0
|
||||
|
||||
snapshot := func() crawlProgress {
|
||||
nodes := make(map[string]pedNode, len(tree))
|
||||
for k, v := range tree {
|
||||
nodes[k] = v
|
||||
}
|
||||
return crawlProgress{Pages: pages, Distinct: countDistinct(tree), MaxGen: maxGen, Nodes: nodes}
|
||||
}
|
||||
|
||||
for len(queue) > 0 {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return tree, err
|
||||
}
|
||||
if pages >= maxCrawlPages || requests >= maxCrawlRequests {
|
||||
log.Printf("pedigree crawl %s: hit cap (pages=%d requests=%d)", hundid, pages, requests)
|
||||
break
|
||||
}
|
||||
item := queue[0]
|
||||
queue = queue[1:]
|
||||
if done[item.hundid] {
|
||||
continue
|
||||
}
|
||||
done[item.hundid] = true
|
||||
|
||||
time.Sleep(skkDelay)
|
||||
pageHTML, pageURL, err := c.fetchPage(ctx, item.hundid)
|
||||
requests += 2
|
||||
if err != nil {
|
||||
if pages == 0 {
|
||||
return tree, err // couldn't even fetch the subject
|
||||
}
|
||||
log.Printf("pedigree crawl %s: fetch %s failed: %v", hundid, item.hundid, err)
|
||||
continue
|
||||
}
|
||||
pages++
|
||||
grid := parseGrid(pageHTML)
|
||||
|
||||
var frontier []struct {
|
||||
gpos uint64
|
||||
ctlid string
|
||||
}
|
||||
for col := 0; col <= crawlGens-1; col++ {
|
||||
cells, ok := grid[col]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for pos, cell := range cells {
|
||||
if !cell.occupied {
|
||||
continue
|
||||
}
|
||||
gpos := item.basePos*(1<<uint(col)) + uint64(pos)
|
||||
key := strconv.FormatUint(gpos, 10)
|
||||
node := tree[key]
|
||||
node.Reg, node.Name, node.Titles = cell.reg, cell.name, cell.titles
|
||||
if col == 0 {
|
||||
node.Hundid = item.hundid
|
||||
}
|
||||
tree[key] = node
|
||||
if g := bitsLen(gpos); g > maxGen {
|
||||
maxGen = g
|
||||
}
|
||||
if col == crawlGens-1 && cell.ctlid != "" {
|
||||
frontier = append(frontier, struct {
|
||||
gpos uint64
|
||||
ctlid string
|
||||
}{gpos, cell.ctlid})
|
||||
}
|
||||
}
|
||||
}
|
||||
report(snapshot())
|
||||
|
||||
for _, f := range frontier {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return tree, err
|
||||
}
|
||||
if requests >= maxCrawlRequests {
|
||||
break
|
||||
}
|
||||
ekey := item.hundid + "|" + f.ctlid
|
||||
hid, ok := edges[ekey]
|
||||
if !ok {
|
||||
time.Sleep(skkDelay)
|
||||
hid, err = c.postbackHundid(ctx, pageHTML, f.ctlid, pageURL)
|
||||
requests++
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
edges[ekey] = hid
|
||||
}
|
||||
if hid != "" {
|
||||
key := strconv.FormatUint(f.gpos, 10)
|
||||
node := tree[key]
|
||||
node.Hundid = hid
|
||||
tree[key] = node
|
||||
queue = append(queue, qitem{hid, f.gpos})
|
||||
}
|
||||
}
|
||||
}
|
||||
report(snapshot())
|
||||
return tree, nil
|
||||
}
|
||||
|
||||
func bitsLen(x uint64) int {
|
||||
n := 0
|
||||
for x > 0 {
|
||||
n++
|
||||
x >>= 1
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ---- job manager ---------------------------------------------------------
|
||||
|
||||
type jobState string
|
||||
|
||||
const (
|
||||
jobRunning jobState = "running"
|
||||
jobDone jobState = "done"
|
||||
jobError jobState = "error"
|
||||
)
|
||||
|
||||
type pedJob struct {
|
||||
id string
|
||||
hundid string
|
||||
subject pedSubject
|
||||
firstPage chan struct{} // closed once the subject's own page is parsed
|
||||
|
||||
mu sync.Mutex
|
||||
state jobState
|
||||
pages int
|
||||
distinct int
|
||||
maxGen int
|
||||
nodes map[string]pedNode
|
||||
errMsg string
|
||||
}
|
||||
|
||||
func (j *pedJob) apply(p crawlProgress) {
|
||||
j.mu.Lock()
|
||||
j.pages, j.distinct, j.maxGen, j.nodes = p.Pages, p.Distinct, p.MaxGen, p.Nodes
|
||||
j.mu.Unlock()
|
||||
}
|
||||
|
||||
type pedManager struct {
|
||||
db *sql.DB
|
||||
|
||||
mu sync.Mutex
|
||||
jobs map[string]*pedJob // keyed by hundid (coalesces duplicate lookups)
|
||||
}
|
||||
|
||||
func newPedManager(db *sql.DB) *pedManager {
|
||||
return &pedManager{db: db, jobs: map[string]*pedJob{}}
|
||||
}
|
||||
|
||||
func (m *pedManager) activeCount() int {
|
||||
n := 0
|
||||
for _, j := range m.jobs {
|
||||
j.mu.Lock()
|
||||
if j.state == jobRunning {
|
||||
n++
|
||||
}
|
||||
j.mu.Unlock()
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// startOrAttach returns the running/finished job for a hundid, or starts a new
|
||||
// background crawl. The boolean reports whether a fresh job was created.
|
||||
func (m *pedManager) startOrAttach(client *skkClient, subject pedSubject) (*pedJob, bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if j, ok := m.jobs[subject.Hundid]; ok {
|
||||
return j, false, nil
|
||||
}
|
||||
if m.activeCount() >= maxActiveJobs {
|
||||
return nil, false, errors.New("busy: too many pedigree lookups in progress, try again shortly")
|
||||
}
|
||||
j := &pedJob{
|
||||
id: subject.Hundid,
|
||||
hundid: subject.Hundid,
|
||||
subject: subject,
|
||||
firstPage: make(chan struct{}),
|
||||
state: jobRunning,
|
||||
nodes: map[string]pedNode{},
|
||||
}
|
||||
m.jobs[subject.Hundid] = j
|
||||
go m.run(client, j)
|
||||
return j, true, nil
|
||||
}
|
||||
|
||||
func (m *pedManager) run(client *skkClient, j *pedJob) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), crawlDeadline)
|
||||
defer cancel()
|
||||
|
||||
firstDone := false
|
||||
report := func(p crawlProgress) {
|
||||
j.apply(p)
|
||||
if !firstDone && p.Pages >= 1 {
|
||||
firstDone = true
|
||||
close(j.firstPage)
|
||||
}
|
||||
}
|
||||
nodes, err := client.crawl(ctx, j.hundid, report)
|
||||
if !firstDone {
|
||||
close(j.firstPage) // unblock waiters even if the very first fetch failed
|
||||
}
|
||||
j.mu.Lock()
|
||||
if err != nil && len(nodes) == 0 {
|
||||
j.state = jobError
|
||||
j.errMsg = err.Error()
|
||||
} else {
|
||||
j.state = jobDone
|
||||
}
|
||||
j.mu.Unlock()
|
||||
|
||||
if len(nodes) > 0 {
|
||||
m.persist(j.subject, nodes)
|
||||
}
|
||||
}
|
||||
|
||||
// snapshot copies a job's current public state under its lock.
|
||||
func (j *pedJob) snapshot() (jobState, int, int, int, map[string]pedNode, string) {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
nodes := make(map[string]pedNode, len(j.nodes))
|
||||
for k, v := range j.nodes {
|
||||
nodes[k] = v
|
||||
}
|
||||
return j.state, j.pages, j.distinct, j.maxGen, nodes, j.errMsg
|
||||
}
|
||||
|
||||
// ---- persistent cache ----------------------------------------------------
|
||||
|
||||
func (m *pedManager) persist(subject pedSubject, nodes map[string]pedNode) {
|
||||
sj, _ := json.Marshal(subject)
|
||||
nj, _ := json.Marshal(nodes)
|
||||
_, err := m.db.Exec(`
|
||||
INSERT INTO pedigree_cache (hundid, subject, nodes, generations, fetched)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(hundid) DO UPDATE SET
|
||||
subject = excluded.subject, nodes = excluded.nodes,
|
||||
generations = excluded.generations, fetched = excluded.fetched`,
|
||||
subject.Hundid, string(sj), string(nj), maxGenerations(nodes), time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
log.Printf("pedigree cache save %s: %v", subject.Hundid, err)
|
||||
}
|
||||
}
|
||||
|
||||
// cached returns a stored tree for a hundid, if present.
|
||||
func (m *pedManager) cached(hundid string) (pedSubject, map[string]pedNode, bool) {
|
||||
var sj, nj string
|
||||
err := m.db.QueryRow(
|
||||
`SELECT subject, nodes FROM pedigree_cache WHERE hundid = ?`, hundid,
|
||||
).Scan(&sj, &nj)
|
||||
if err != nil {
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
log.Printf("pedigree cache get %s: %v", hundid, err)
|
||||
}
|
||||
return pedSubject{}, nil, false
|
||||
}
|
||||
var subject pedSubject
|
||||
var nodes map[string]pedNode
|
||||
json.Unmarshal([]byte(sj), &subject)
|
||||
json.Unmarshal([]byte(nj), &nodes)
|
||||
return subject, nodes, true
|
||||
}
|
||||
|
||||
func maxGenerations(nodes map[string]pedNode) int {
|
||||
max := 0
|
||||
for k := range nodes {
|
||||
if p, err := strconv.ParseUint(k, 10, 64); err == nil {
|
||||
if g := bitsLen(p); g > max {
|
||||
max = g
|
||||
}
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
// ---- HTTP handlers -------------------------------------------------------
|
||||
|
||||
type pedLookupResponse struct {
|
||||
Status string `json:"status"` // done | running | choose
|
||||
JobID string `json:"jobId,omitempty"`
|
||||
Hundid string `json:"hundid,omitempty"`
|
||||
Subject *pedSubject `json:"subject,omitempty"`
|
||||
Generations int `json:"generations,omitempty"`
|
||||
Nodes map[string]pedNode `json:"nodes,omitempty"`
|
||||
Matches []hundDataRow `json:"matches,omitempty"`
|
||||
}
|
||||
|
||||
func rowToSubject(r hundDataRow) pedSubject {
|
||||
return pedSubject{
|
||||
Hundid: r.Hundid,
|
||||
Reg: strings.TrimSpace(r.Regnr),
|
||||
Name: strings.TrimSpace(r.Hundnamn),
|
||||
Breed: strings.TrimSpace(r.Rastext),
|
||||
Chip: strings.TrimSpace(r.Chipnr),
|
||||
Sex: strings.TrimSpace(r.Kon),
|
||||
}
|
||||
}
|
||||
|
||||
// handleLookup resolves a query and returns a cached tree, a disambiguation list,
|
||||
// or an immediate 7-generation tree with a background job crawling deeper.
|
||||
func (m *pedManager) handleLookup(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Q string `json:"q"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||
http.Error(w, "bad json: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
q := strings.TrimSpace(req.Q)
|
||||
if q == "" {
|
||||
http.Error(w, "empty query", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
client, err := newSKKClient()
|
||||
if err != nil {
|
||||
http.Error(w, "server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
ctx := r.Context()
|
||||
if err := client.warm(ctx); err != nil {
|
||||
http.Error(w, "upstream unavailable", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
rows, err := client.resolve(ctx, q)
|
||||
if err != nil {
|
||||
http.Error(w, "lookup failed: "+err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
rows = withHundid(rows)
|
||||
switch {
|
||||
case len(rows) == 0:
|
||||
http.Error(w, "no dog found for "+q, http.StatusNotFound)
|
||||
return
|
||||
case len(rows) > 1:
|
||||
writeJSON(w, pedLookupResponse{Status: "choose", Matches: rows})
|
||||
return
|
||||
}
|
||||
subject := rowToSubject(rows[0])
|
||||
|
||||
if subj, nodes, ok := m.cached(subject.Hundid); ok {
|
||||
writeJSON(w, pedLookupResponse{
|
||||
Status: "done", Hundid: subj.Hundid, Subject: &subj,
|
||||
Generations: maxGenerations(nodes), Nodes: nodes,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
job, _, err := m.startOrAttach(client, subject)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-job.firstPage:
|
||||
case <-time.After(firstPageWait):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
state, _, _, gen, nodes, msg := job.snapshot()
|
||||
if state == jobError {
|
||||
http.Error(w, "pedigree fetch failed: "+msg, http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
status := "running"
|
||||
if state == jobDone {
|
||||
status = "done"
|
||||
}
|
||||
writeJSON(w, pedLookupResponse{
|
||||
Status: status, JobID: job.id, Hundid: subject.Hundid,
|
||||
Subject: &subject, Generations: gen, Nodes: nodes,
|
||||
})
|
||||
}
|
||||
|
||||
type pedStatusResponse struct {
|
||||
Status string `json:"status"`
|
||||
Pages int `json:"pages"`
|
||||
Distinct int `json:"distinct"`
|
||||
Generations int `json:"generations"`
|
||||
Nodes map[string]pedNode `json:"nodes,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// handleStatus returns a running crawl's current partial tree so the client can
|
||||
// fill the view in progressively, and the final tree when it finishes.
|
||||
func (m *pedManager) handleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.URL.Query().Get("job")
|
||||
m.mu.Lock()
|
||||
job := m.jobs[id]
|
||||
m.mu.Unlock()
|
||||
if job == nil {
|
||||
// A finished job may have been evicted, but the cache still has the tree.
|
||||
if _, nodes, ok := m.cached(id); ok {
|
||||
writeJSON(w, pedStatusResponse{
|
||||
Status: "done", Generations: maxGenerations(nodes),
|
||||
Distinct: countDistinct(nodes), Nodes: nodes, Pages: 0,
|
||||
})
|
||||
return
|
||||
}
|
||||
http.Error(w, "unknown job", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
state, pages, distinct, gen, nodes, msg := job.snapshot()
|
||||
writeJSON(w, pedStatusResponse{
|
||||
Status: string(state), Pages: pages, Distinct: distinct,
|
||||
Generations: gen, Nodes: nodes, Error: msg,
|
||||
})
|
||||
}
|
||||
|
||||
// countDistinct counts unique ancestors, keyed by registration number (falling
|
||||
// back to name). Pedigree collapse means one dog can fill many positions, so
|
||||
// this is smaller than the number of occupied positions.
|
||||
func countDistinct(nodes map[string]pedNode) int {
|
||||
seen := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
k := n.Reg
|
||||
if k == "" {
|
||||
k = n.Name
|
||||
}
|
||||
if k != "" {
|
||||
seen[k] = true
|
||||
}
|
||||
}
|
||||
return len(seen)
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
// withHundid drops resolver rows lacking a usable hundid (defensive).
|
||||
func withHundid(rows []hundDataRow) []hundDataRow {
|
||||
out := rows[:0]
|
||||
for _, r := range rows {
|
||||
if strings.TrimSpace(r.Hundid) != "" {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user