diff --git a/hosts/manatee/modules/restic/default.nix b/hosts/manatee/modules/restic/default.nix index 86dfc69..61974cb 100644 --- a/hosts/manatee/modules/restic/default.nix +++ b/hosts/manatee/modules/restic/default.nix @@ -1,6 +1,121 @@ -{ lib, config, ... }: +{ + pkgs, + lib, + config, + ... +}: let enabled = config.mod.restic.enable; + + # Where the consistent, restic-ready copies of service state are staged. + # Lives on the external backup disk. root-only (0700): these dumps contain + # credentials (HA .storage, secrets.yaml, session tokens, the romm database). + stateDir = "/mnt/backup/public/state"; + + # Produce consistent, restore-ready copies of every service's state under + # ${stateDir}. Runs as root before the state backups. SQLite DBs are copied + # via the online-backup API (safe on a live, WAL-mode DB); the romm MariaDB is + # dumped logically with mariadb-dump --single-transaction. Rebuildable caches + # and indexes are skipped to keep the offsite copy lean. + stateDumpScript = pkgs.writeShellScript "state-backup-dump" '' + export PATH=${ + lib.makeBinPath [ + pkgs.sqlite + pkgs.rsync + pkgs.podman + pkgs.coreutils + pkgs.findutils + ] + } + set -euo pipefail + umask 077 + + STATE=${lib.escapeShellArg stateDir} + install -d -m700 "$STATE" + + # snapshot SRC DEST [extra rsync excludes...] + # Copies plain files with rsync (excluding live DBs, their WAL/SHM sidecars + # and logs), then makes a consistent online copy of each *.db / *.sqlite. + snapshot() { + local src="$1" dst="$2" + shift 2 + if [ ! -e "$src" ]; then + echo "state-backup: skip (missing) $src" + return 0 + fi + install -d -m700 "$dst" + rsync -a --delete \ + --exclude='*.sqlite' --exclude='*.sqlite-shm' --exclude='*.sqlite-wal' \ + --exclude='*.db' --exclude='*.db-shm' --exclude='*.db-wal' \ + --exclude='*.log' --exclude='*.log.*' \ + --exclude='__pycache__/' \ + "$@" \ + "$src/" "$dst/" + while IFS= read -r db; do + local rel="''${db#"$src"/}" + install -d -m700 "$dst/$(dirname "$rel")" + sqlite3 "$db" ".backup '$dst/$rel'" + done < <(find "$src" -maxdepth 4 -type f \( -name '*.sqlite' -o -name '*.db' \)) + } + + # --- romm: logical MariaDB dump + user assets (saves/states/screenshots) --- + # Cover art under resources/ is intentionally skipped (refetched from + # metadata providers); redis is a rebuildable cache. + if podman exec romm-db true 2>/dev/null; then + install -d -m700 "$STATE/romm" + podman exec romm-db sh -c \ + 'exec mariadb-dump --user="$MARIADB_USER" --password="$MARIADB_PASSWORD" --single-transaction --no-tablespaces "$MARIADB_DATABASE"' \ + > "$STATE/romm/romm.sql" + rsync -a --delete /var/lib/romm/assets/ "$STATE/romm/assets/" + else + echo "state-backup: skip (romm-db not running)" + fi + + # --- komga: only database.sqlite matters (read progress, users, collections). + # tasks.sqlite is transient and the lucene/ index is rebuilt on demand. --- + if [ -e /var/lib/komga/database.sqlite ]; then + install -d -m700 "$STATE/komga" + sqlite3 /var/lib/komga/database.sqlite ".backup '$STATE/komga/database.sqlite'" + fi + + # --- home-assistant: config, credentials and integrations, plus the two live + # SQLite DBs. Recorder history (home-assistant_v2.db) is included per choice. + # deps/ and tts/ are reinstalled/regenerated; logs are dropped. --- + HA=/home/alex/.config/home-assistant + if [ -e "$HA" ]; then + install -d -m700 "$STATE/home-assistant" + rsync -a --delete \ + --exclude='home-assistant_v2.db' --exclude='home-assistant_v2.db-shm' --exclude='home-assistant_v2.db-wal' \ + --exclude='zigbee.db' --exclude='zigbee.db-shm' --exclude='zigbee.db-wal' \ + --exclude='*.log' --exclude='*.log.*' --exclude='*.log.fault' \ + --exclude='deps/' --exclude='tts/' --exclude='__pycache__/' \ + "$HA/" "$STATE/home-assistant/" + for db in home-assistant_v2.db zigbee.db; do + if [ -e "$HA/$db" ]; then + sqlite3 "$HA/$db" ".backup '$STATE/home-assistant/$db'" + fi + done + fi + + # --- remaining NixOS services: generic snapshot, skipping rebuildable caches --- + snapshot /var/lib/navidrome "$STATE/navidrome" --exclude='cache/' + snapshot /var/lib/jellyfin "$STATE/jellyfin" --exclude='transcodes/' --exclude='cache/' --exclude='metadata/' --exclude='log/' --exclude='data/subtitles/' + snapshot /var/lib/audiobookshelf "$STATE/audiobookshelf" --exclude='metadata/' + snapshot /var/lib/puppy-tracker "$STATE/puppy-tracker" + snapshot /var/lib/solo-referee "$STATE/solo-referee" + snapshot /var/lib/todo "$STATE/todo" + snapshot /var/lib/komga-book-manager "$STATE/komga-book-manager" + snapshot /var/lib/komga-reading-stats "$STATE/komga-reading-stats" + + echo "state-backup: dump complete" + ''; + + # Retention shared by all jobs. + pruneOpts = [ + "--keep-daily 1" + "--keep-weekly 7" + "--keep-yearly 12" + ]; in { options.mod.restic.enable = lib.mkEnableOption "Enable restic"; @@ -22,11 +137,7 @@ in OnCalendar = "*-*-* 0/12:00:00"; Persistent = true; }; - pruneOpts = [ - "--keep-daily 1" - "--keep-weekly 7" - "--keep-yearly 12" - ]; + inherit pruneOpts; }; "sync-to-cloud" = { @@ -39,14 +150,57 @@ in OnCalendar = "*-*-* 0/12:00:00"; Persistent = true; }; - pruneOpts = [ - "--keep-daily 1" - "--keep-weekly 7" - "--keep-yearly 12" + inherit pruneOpts; + }; + + # Service state (databases etc.). No timer of their own: the state-backup + # orchestrator below runs the dump once and then triggers these in order, + # so both back up the same consistent staging dir without racing it. + "state-to-external" = { + initialize = true; + passwordFile = config.age.secrets.restic-password.path; + paths = [ stateDir ]; + repository = "/mnt/backup/restic"; + timerConfig = null; + inherit pruneOpts; + }; + + "state-to-cloud" = { + initialize = true; + passwordFile = config.age.secrets.restic-password.path; + environmentFile = config.age.secrets.restic-cloud-sync-key.path; + repositoryFile = config.age.secrets.restic-cloud-sync-repository.path; + paths = [ stateDir ]; + timerConfig = null; + inherit pruneOpts; + }; + }; + + # Dump service state, then push it to the external disk and the cloud, in + # that order. A "-" prefix means a failure of one restic target does not + # block the other; a failed dump (no prefix) aborts before either runs. + systemd.services.state-backup = { + description = "Dump service state and back it up (external + cloud)"; + after = [ "podman.service" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = [ + "${stateDumpScript}" + "-${pkgs.systemd}/bin/systemctl start --wait restic-backups-state-to-external.service" + "-${pkgs.systemd}/bin/systemctl start --wait restic-backups-state-to-cloud.service" ]; }; }; + systemd.timers.state-backup = { + description = "Timer for service-state backups"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "*-*-* 0/12:00:00"; + Persistent = true; + }; + }; + age.secrets = { "restic-password".file = ../../../../secrets/manatee/restic-password.age; "restic-cloud-sync-key".file = ../../../../secrets/manatee/restic-cloud-sync-key.age;