diff --git a/README.md b/README.md index 29f0483..a175650 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ source-of-truth and sync between devices. client POSTs its full event list to `/api/events/sync`. The server merges it with its own copy using last-write-wins on `updatedAt` and returns the merged set. +- The server keeps its copy in a SQLite database (`puppy.db`); events and the + shared profile are separate tables, and last-write-wins is enforced by the + upsert itself. On first start it auto-imports any legacy `events.json` / + `config.json` sitting alongside it, renaming them to `*.imported`. - Service worker bypasses cache for `/api/*` so writes always hit the server when online; static assets are still cached for offline use. - The puppy's name and birthday are a shared profile stored on the host @@ -35,7 +39,8 @@ puppy-tracker/ ├── module.nix # systemd unit, StateDirectory, hardening ├── server/ │ ├── go.mod -│ └── main.go # JSON-file store, LWW sync, static file serving +│ ├── go.sum +│ └── main.go # SQLite store, LWW sync, static file serving └── src/ # the web app ├── index.html ├── app.js @@ -52,7 +57,7 @@ nix run # http://localhost:8080, data in $XDG_DATA_HOME/pu PUPPY_ADDR=:9000 nix run # custom port # Hot-iterate (data in /tmp): -nix develop -c sh -c 'cd server && go run . -static ../src -data /tmp/puppy-events.json' +nix develop -c sh -c 'cd server && go run . -static ../src -data /tmp/puppy.db' ``` ## Use it on NixOS @@ -81,8 +86,9 @@ In your system flake: } ``` -The server runs as a `DynamicUser` systemd unit. Data is stored at -`/var/lib/puppy-tracker/events.json` via `StateDirectory`. +The server runs as a `DynamicUser` systemd unit. Data is stored in a SQLite +database at `/var/lib/puppy-tracker/puppy.db` via `StateDirectory` (with photos +alongside it under `photos/`). ## Notes diff --git a/flake.nix b/flake.nix index 736db27..5108df4 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,7 @@ pname = "puppy-tracker-server"; version = "0.2.0"; src = ./server; - vendorHash = null; # no external dependencies + vendorHash = "sha256-fqXpr9fV1jeT7503uAjb4jjNPlXfESFnXr7/Uc83d4o="; # Pure-Go build for a tiny static binary. env.CGO_ENABLED = "0"; ldflags = [ "-s" "-w" ]; @@ -76,7 +76,7 @@ exec ${server}/bin/puppy-tracker-server \ -addr "''${PUPPY_ADDR:-:8080}" \ -static ${static}/share/puppy-tracker \ - -data "$data_dir/events.json" + -data "$data_dir/puppy.db" ''); meta.description = "Run puppy-tracker locally (data in $XDG_DATA_HOME/puppy-tracker)"; }; diff --git a/module.nix b/module.nix index 4cd22eb..41a2402 100644 --- a/module.nix +++ b/module.nix @@ -53,7 +53,7 @@ in "${cfg.package}/bin/puppy-tracker-server" "-addr ${cfg.address}:${toString cfg.port}" "-static ${cfg.staticPackage}/share/puppy-tracker" - "-data /var/lib/puppy-tracker/events.json" + "-data /var/lib/puppy-tracker/puppy.db" ]; DynamicUser = true; diff --git a/server/go.mod b/server/go.mod index 681a6a6..bdf8719 100644 --- a/server/go.mod +++ b/server/go.mod @@ -1,3 +1,17 @@ module puppy-tracker -go 1.22 +go 1.25.0 + +require modernc.org/sqlite v1.53.0 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.44.0 // indirect + modernc.org/libc v1.73.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/server/go.sum b/server/go.sum new file mode 100644 index 0000000..b054032 --- /dev/null +++ b/server/go.sum @@ -0,0 +1,51 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= +modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= +modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= +modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= +modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/server/main.go b/server/main.go index 9e3e6bf..9d53de6 100644 --- a/server/main.go +++ b/server/main.go @@ -1,6 +1,7 @@ package main import ( + "database/sql" "encoding/json" "errors" "flag" @@ -12,8 +13,9 @@ import ( "path/filepath" "regexp" "strings" - "sync" "time" + + _ "modernc.org/sqlite" ) type Event struct { @@ -45,155 +47,230 @@ type Config struct { } type ConfigStore struct { - path string - mu sync.Mutex - cfg Config + db *sql.DB } -func newConfigStore(path string) (*ConfigStore, error) { - cs := &ConfigStore{path: path} - f, err := os.Open(path) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return cs, nil - } - return nil, err - } - defer f.Close() - if err := json.NewDecoder(f).Decode(&cs.cfg); err != nil && !errors.Is(err, io.EOF) { - return nil, err - } - return cs, nil +func newConfigStore(db *sql.DB) *ConfigStore { + return &ConfigStore{db: db} } func (cs *ConfigStore) get() Config { - cs.mu.Lock() - defer cs.mu.Unlock() - return cs.cfg + var c Config + // The profile lives in a single row (id = 1). A missing row is the + // pre-configuration state, so a zero-value Config is the right answer. + err := cs.db.QueryRow( + `SELECT name, birthday, updated FROM config WHERE id = 1`, + ).Scan(&c.Name, &c.Birthday, &c.UpdatedAt) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + log.Printf("config get: %v", err) + } + return c } // merge applies an incoming config with last-write-wins by UpdatedAt and // returns the resulting stored config (which the caller sends back). func (cs *ConfigStore) merge(in Config) (Config, error) { - cs.mu.Lock() - defer cs.mu.Unlock() - if in.UpdatedAt > cs.cfg.UpdatedAt { - cs.cfg = in - if err := cs.saveLocked(); err != nil { - return cs.cfg, err - } - } - return cs.cfg, nil -} - -// Caller must hold cs.mu. -func (cs *ConfigStore) saveLocked() error { - if err := os.MkdirAll(filepath.Dir(cs.path), 0o755); err != nil { - return err - } - tmp := cs.path + ".tmp" - f, err := os.Create(tmp) + // The upsert's WHERE clause enforces last-write-wins: the incoming row only + // replaces the stored one when it is strictly newer. + _, err := cs.db.Exec(` + INSERT INTO config (id, name, birthday, updated) + VALUES (1, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + name = excluded.name, birthday = excluded.birthday, updated = excluded.updated + WHERE excluded.updated > config.updated`, + in.Name, in.Birthday, in.UpdatedAt) if err != nil { - return err + return Config{}, err } - enc := json.NewEncoder(f) - enc.SetIndent("", " ") - if err := enc.Encode(cs.cfg); err != nil { - f.Close() - os.Remove(tmp) - return err - } - if err := f.Close(); err != nil { - return err - } - return os.Rename(tmp, cs.path) + return cs.get(), nil } type Store struct { - path string - mu sync.Mutex - data map[string]Event + db *sql.DB } -func newStore(path string) (*Store, error) { - s := &Store{path: path, data: map[string]Event{}} - if err := s.load(); err != nil { +func newStore(db *sql.DB) *Store { + return &Store{db: db} +} + +// sync merges client events into the store using last-write-wins by UpdatedAt, +// then returns the full merged set (including tombstones, which must propagate). +func (s *Store) sync(client []Event) ([]Event, error) { + tx, err := s.db.Begin() + if err != nil { return nil, err } - return s, nil + defer tx.Rollback() + + // The WHERE clause on the upsert is the last-write-wins rule: an incoming + // event only overwrites the stored one when its updatedAt is strictly newer. + stmt, err := tx.Prepare(` + INSERT INTO events (id, type, at, note, photo_id, weight, updated, deleted) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + type = excluded.type, at = excluded.at, note = excluded.note, + photo_id = excluded.photo_id, weight = excluded.weight, + updated = excluded.updated, deleted = excluded.deleted + WHERE excluded.updated > events.updated`) + if err != nil { + return nil, err + } + defer stmt.Close() + + for _, ce := range client { + if ce.ID == "" { + continue + } + if _, err := stmt.Exec( + ce.ID, ce.Type, ce.At, ce.Note, ce.PhotoID, ce.Weight, ce.UpdatedAt, ce.Deleted, + ); err != nil { + return nil, err + } + } + if err := tx.Commit(); err != nil { + return nil, err + } + return s.all() } -func (s *Store) load() error { - f, err := os.Open(s.path) +// all returns every stored event, tombstones included. +func (s *Store) all() ([]Event, error) { + rows, err := s.db.Query( + `SELECT id, type, at, note, photo_id, weight, updated, deleted FROM events`) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]Event, 0) + for rows.Next() { + var e Event + if err := rows.Scan( + &e.ID, &e.Type, &e.At, &e.Note, &e.PhotoID, &e.Weight, &e.UpdatedAt, &e.Deleted, + ); err != nil { + return nil, err + } + out = append(out, e) + } + return out, rows.Err() +} + +// openDB opens (creating if needed) the SQLite database and ensures the schema +// exists. WAL mode plays nicely with concurrent readers during a sync write; +// busy_timeout avoids spurious "database is locked" errors under contention. +func openDB(path string) (*sql.DB, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, err + } + for _, pragma := range []string{ + `PRAGMA journal_mode = WAL`, + `PRAGMA busy_timeout = 5000`, + `PRAGMA synchronous = NORMAL`, + } { + if _, err := db.Exec(pragma); err != nil { + db.Close() + return nil, err + } + } + schema := ` + CREATE TABLE IF NOT EXISTS events ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL DEFAULT '', + at INTEGER NOT NULL DEFAULT 0, + note TEXT NOT NULL DEFAULT '', + photo_id TEXT NOT NULL DEFAULT '', + weight REAL NOT NULL DEFAULT 0, + updated INTEGER NOT NULL DEFAULT 0, + deleted INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE IF NOT EXISTS config ( + id INTEGER PRIMARY KEY CHECK (id = 1), + name TEXT NOT NULL DEFAULT '', + birthday TEXT NOT NULL DEFAULT '', + updated INTEGER NOT NULL DEFAULT 0 + );` + if _, err := db.Exec(schema); err != nil { + db.Close() + return nil, err + } + return db, nil +} + +// migrateJSON imports a pre-SQLite events.json / config.json sitting in dataDir +// into an otherwise-empty database, then renames each file to *.imported so the +// import runs exactly once. It is a no-op when the DB already holds data or the +// legacy files are absent. +func migrateJSON(db *sql.DB, dataDir string) error { + if err := importEvents(db, filepath.Join(dataDir, "events.json")); err != nil { + return err + } + return importConfig(db, filepath.Join(dataDir, "config.json")) +} + +func importEvents(db *sql.DB, path string) error { + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM events`).Scan(&n); err != nil { + return err + } + if n > 0 { + return nil // DB already has data; never clobber it + } + f, err := os.Open(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil } return err } - defer f.Close() var evs []Event - if err := json.NewDecoder(f).Decode(&evs); err != nil { - if errors.Is(err, io.EOF) { + dec := json.NewDecoder(f) + err = dec.Decode(&evs) + f.Close() + if err != nil && !errors.Is(err, io.EOF) { + return err + } + store := newStore(db) + if _, err := store.sync(evs); err != nil { + return err + } + log.Printf("migrated %d events from %s", len(evs), path) + return os.Rename(path, path+".imported") +} + +func importConfig(db *sql.DB, path string) error { + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM config`).Scan(&n); err != nil { + return err + } + if n > 0 { + return nil + } + f, err := os.Open(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { return nil } return err } - for _, e := range evs { - s.data[e.ID] = e - } - return nil -} - -// Caller must hold s.mu. -func (s *Store) saveLocked() error { - if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil { + var c Config + dec := json.NewDecoder(f) + err = dec.Decode(&c) + f.Close() + if err != nil && !errors.Is(err, io.EOF) { return err } - tmp := s.path + ".tmp" - f, err := os.Create(tmp) - if err != nil { + if c.UpdatedAt == 0 { + // Nothing meaningful to import; leave config empty. + return os.Rename(path, path+".imported") + } + if _, err := newConfigStore(db).merge(c); err != nil { return err } - evs := make([]Event, 0, len(s.data)) - for _, e := range s.data { - evs = append(evs, e) - } - enc := json.NewEncoder(f) - enc.SetIndent("", " ") - if err := enc.Encode(evs); err != nil { - f.Close() - os.Remove(tmp) - return err - } - if err := f.Close(); err != nil { - return err - } - return os.Rename(tmp, s.path) -} - -// sync merges client events into the store using last-write-wins by UpdatedAt, -// then returns the full merged set. -func (s *Store) sync(client []Event) ([]Event, error) { - s.mu.Lock() - defer s.mu.Unlock() - for _, ce := range client { - if ce.ID == "" { - continue - } - existing, ok := s.data[ce.ID] - if !ok || ce.UpdatedAt > existing.UpdatedAt { - s.data[ce.ID] = ce - } - } - if err := s.saveLocked(); err != nil { - return nil, err - } - out := make([]Event, 0, len(s.data)) - for _, e := range s.data { - out = append(out, e) - } - return out, nil + log.Printf("migrated config from %s", path) + return os.Rename(path, path+".imported") } type syncRequest struct { @@ -213,19 +290,23 @@ func (c cacheControlFS) Open(name string) (http.File, error) { return c.root.Ope func main() { addr := flag.String("addr", ":8080", "listen address (e.g. :8080 or 0.0.0.0:8080)") - dataPath := flag.String("data", "events.json", "path to events JSON file") + dataPath := flag.String("data", "puppy.db", "path to SQLite database file") staticDir := flag.String("static", "", "directory of static files to serve") flag.Parse() - store, err := newStore(*dataPath) + db, err := openDB(*dataPath) if err != nil { - log.Fatalf("load store: %v", err) + log.Fatalf("open db: %v", err) + } + defer db.Close() + + // One-time import of any pre-SQLite JSON data sitting next to the DB. + if err := migrateJSON(db, filepath.Dir(*dataPath)); err != nil { + log.Fatalf("migrate json: %v", err) } - configStore, err := newConfigStore(filepath.Join(filepath.Dir(*dataPath), "config.json")) - if err != nil { - log.Fatalf("load config: %v", err) - } + store := newStore(db) + configStore := newConfigStore(db) photosDir := filepath.Join(filepath.Dir(*dataPath), "photos") if err := os.MkdirAll(photosDir, 0o755); err != nil {