manatee: Update monitoring in home-assistant

- Add all disks to smartd
- Generate home-assistant config in nix
- Add metrics for all HDDs

hosts/manatee/disk-config.nix

@@ -79,11 +79,16 @@
         { device = config.disko.devices.disk.root.device; }
         { device = config.disko.devices.disk.disk1.device; }
         { device = config.disko.devices.disk.disk2.device; }
+        { device = config.disko.devices.disk.disk3.device; }
+        { device = config.disko.devices.disk.disk4.device; }
       ];
     };
 
     services.zfs.autoScrub.enable = true;
 
+    # Don't force-import the pool if it appears in use elsewhere; safer default in 26.11+.
+    boot.zfs.forceImportRoot = false;
+
     networking.hostId = "0a9474e7"; # Required by ZFS
     disko.devices = {
       disk = {

hosts/manatee/modules/default.nix

@@ -23,6 +23,7 @@ in
       komga.enable = true;
       romm.enable = true;
       homepage.enable = true;
+      disk-smart.enable = true;
     };
   };
 }

hosts/manatee/modules/disk-smart/default.nix

@@ -0,0 +1,159 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  enabled = config.mod.disk-smart.enable;
+
+  disks = [
+    { path = "/dev/disk/by-id/ata-ST8000VN004-3CP101_WWZ8QCG4"; name = "seagate_8tb_1"; label = "Seagate 8TB #1"; }
+    { path = "/dev/disk/by-id/ata-ST8000VN004-3CP101_WWZ8QDJ5"; name = "seagate_8tb_2"; label = "Seagate 8TB #2"; }
+    { path = "/dev/disk/by-id/ata-TOSHIBA_MG10ACA20TE_85K2A0UCF4MJ"; name = "toshiba_20tb_1"; label = "Toshiba 20TB #1"; }
+    { path = "/dev/disk/by-id/ata-TOSHIBA_MG10ACA20TE_85K2A0V6F4MJ"; name = "toshiba_20tb_2"; label = "Toshiba 20TB #2"; }
+  ];
+
+  outputDir = "/var/lib/disk-smart";
+
+  collectScript = pkgs.writeShellScript "disk-smart-collect" ''
+    set -euo pipefail
+    export PATH="${lib.makeBinPath [ pkgs.smartmontools pkgs.jq pkgs.coreutils ]}"
+
+    mkdir -p ${outputDir}
+
+    result="{"
+
+    ${lib.concatMapStringsSep "\n" (disk: ''
+      raw=$(smartctl -j -A -H ${disk.path} 2>/dev/null || true)
+
+      temp=$(echo "$raw" | jq -r '.temperature.current // empty')
+      power_on=$(echo "$raw" | jq -r '.power_on_time.hours // empty')
+      smart_status=$(echo "$raw" | jq -r '.smart_status.passed // empty')
+      reallocated=$(echo "$raw" | jq -r '[.ata_smart_attributes.table[] | select(.name == "Reallocated_Sector_Ct")][0].raw.value // empty')
+      pending=$(echo "$raw" | jq -r '[.ata_smart_attributes.table[] | select(.name == "Current_Pending_Sector")][0].raw.value // empty')
+
+      result="$result\"${disk.name}\":{\"temperature\":$temp,\"power_on_hours\":$power_on,\"smart_passed\":$smart_status,\"reallocated_sectors\":$reallocated,\"pending_sectors\":$pending},"
+    '') disks}
+
+    # Remove trailing comma, close object
+    result="''${result%,}}"
+
+    echo "$result" | jq . > ${outputDir}/smart.json.tmp
+    mv ${outputDir}/smart.json.tmp ${outputDir}/smart.json
+  '';
+
+  indent = prefix: s:
+    lib.concatMapStringsSep "\n"
+      (line: if line == "" then line else prefix + line)
+      (lib.splitString "\n" s);
+
+  mkSensor = disk: ''
+- name: "${disk.label} Temperature"
+  value_template: "{{ value_json.${disk.name}.temperature }}"
+  unit_of_measurement: "°C"
+  device_class: temperature
+  state_class: measurement
+- name: "${disk.label} Power On Hours"
+  value_template: "{{ value_json.${disk.name}.power_on_hours }}"
+  unit_of_measurement: "h"
+  state_class: total_increasing
+- name: "${disk.label} SMART Passed"
+  value_template: "{{ value_json.${disk.name}.smart_passed }}"
+- name: "${disk.label} Reallocated Sectors"
+  value_template: "{{ value_json.${disk.name}.reallocated_sectors }}"
+  state_class: measurement
+- name: "${disk.label} Pending Sectors"
+  value_template: "{{ value_json.${disk.name}.pending_sectors }}"
+  state_class: measurement
+'';
+
+  sensorYaml = indent "      " (lib.concatMapStrings mkSensor disks);
+
+  sectorEntities = lib.concatMap (disk: [
+    "sensor.${disk.name}_reallocated_sectors"
+    "sensor.${disk.name}_pending_sectors"
+  ]) disks;
+
+  sectorEntitiesYaml = lib.concatMapStringsSep "\n"
+    (id: "          - ${id}") sectorEntities;
+
+  smartPassedEntities = map (disk: "sensor.${disk.name}_smart_passed") disks;
+
+  smartPassedEntitiesYaml = lib.concatMapStringsSep "\n"
+    (id: "          - ${id}") smartPassedEntities;
+in
+{
+  options = {
+    mod.disk-smart = {
+      enable = lib.mkEnableOption "Enable disk SMART monitoring module";
+    };
+  };
+
+  config = lib.mkIf enabled {
+    mod.home-assistant.extraConfig = ''
+rest:
+  - resource: http://127.0.0.1:9633/smart.json
+    scan_interval: 60
+    sensor:
+${sensorYaml}
+automation disk_smart:
+  - alias: "Disk sector count increased"
+    trigger:
+      - platform: state
+        entity_id:
+${sectorEntitiesYaml}
+    condition:
+      - condition: template
+        value_template: "{{ trigger.from_state.state | int(-1) >= 0 and trigger.to_state.state | int(0) > trigger.from_state.state | int(0) }}"
+    action:
+      - service: notify.mobile_app_pixel_9_pro
+        data:
+          title: "Disk SMART warning"
+          message: "{{ trigger.to_state.attributes.friendly_name }} increased from {{ trigger.from_state.state }} to {{ trigger.to_state.state }}"
+  - alias: "Disk SMART check failed"
+    trigger:
+      - platform: state
+        entity_id:
+${smartPassedEntitiesYaml}
+    condition:
+      - condition: template
+        value_template: "{{ trigger.to_state.state | lower == 'false' }}"
+    action:
+      - service: notify.mobile_app_pixel_9_pro
+        data:
+          title: "Disk SMART FAILURE"
+          message: "{{ trigger.to_state.attributes.friendly_name }} reports SMART failure — drive is likely failing"
+'';
+
+    systemd.services.disk-smart-collect = {
+      description = "Collect disk SMART data";
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = collectScript;
+      };
+    };
+
+    systemd.timers.disk-smart-collect = {
+      description = "Periodically collect disk SMART data";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnBootSec = "1min";
+        OnUnitActiveSec = "1min";
+      };
+    };
+
+    services.nginx.virtualHosts."127.0.0.1" = {
+      listen = [
+        { addr = "127.0.0.1"; port = 9633; }
+      ];
+
+      locations."= /smart.json" = {
+        alias = "${outputDir}/smart.json";
+        extraConfig = ''
+          default_type application/json;
+        '';
+      };
+    };
+  };
+}

hosts/manatee/modules/home-assistant/default.nix

@@ -6,8 +6,42 @@
 }:
 let
   nginxEnabled = config.mod.nginx.enable;
+  cfg = config.mod.home-assistant;
 
-  script = pkgs.writeShellScript "bt-reset" ''
+  configFile = pkgs.writeText "ha-configuration.yaml" ''
+# Loads default set of integrations. Do not remove.
+default_config:
+
+http:
+  use_x_forwarded_for: true
+  trusted_proxies:
+    - 127.0.0.1
+
+# Load frontend themes from the themes folder
+frontend:
+  themes: !include_dir_merge_named themes
+
+automation: !include automations.yaml
+script: !include scripts.yaml
+scene: !include scenes.yaml
+
+recorder:
+  purge_keep_days: 365
+
+alert:
+  fridge_door:
+    name: Fridge is open
+    done_message: Fride is closed
+    entity_id: binary_sensor.kyldorr
+    state: "on"
+    repeat: 2
+    skip_first: true
+    notifiers:
+      - mobile_app_pixel_9_pro
+
+${cfg.extraConfig}'';
+
+  btResetScript = pkgs.writeShellScript "bt-reset" ''
     set -euo pipefail
     export PATH="${
       lib.makeBinPath [
@@ -62,181 +96,194 @@ let
   '';
 in
 {
-  mod.homepage.services = [
+  options = {
-    {
+    mod.home-assistant = {
-      name = "Home Assistant";
+      extraConfig = lib.mkOption {
-      port = 8123;
+        type = lib.types.lines;
-      description = "Home automation";
+        default = "";
-    }
+        description = "Extra YAML to append to Home Assistant's configuration.yaml";
-  ];
+      };
-
-  hardware.bluetooth.enable = true;
-
-  virtualisation.oci-containers = {
-    backend = "podman";
-
-    containers.homeassistant = {
-      image = "ghcr.io/home-assistant/home-assistant:stable";
-
-      volumes = [
-        "/home/alex/.config/home-assistant:/config"
-        # Pass in bluetooth
-        "/run/dbus:/run/dbus:ro"
-      ];
-
-      environment.TZ = "Europe/Stockholm";
-
-      extraOptions = [
-        "--network=host"
-
-        # Allows HA to perform low-level network operations (scan/reset adapter)
-        "--cap-add=NET_ADMIN"
-        "--cap-add=NET_RAW"
-
-        # Pass in Zigbee antenna
-        "--device=/dev/serial/by-id/usb-Nabu_Casa_ZBT-2_9C139EAAD464-if00:/dev/ttyACM0"
-      ];
     };
   };
 
-  services = {
+  config = {
-    blueman.enable = true;
+    mod.homepage.services = [
+      {
+        name = "Home Assistant";
+        port = 8123;
+        description = "Home automation";
+      }
+    ];
 
-    nginx = lib.mkIf nginxEnabled {
+    hardware.bluetooth.enable = true;
-      recommendedProxySettings = true;
 
-      virtualHosts."ha.ppp.pm" = {
+    virtualisation.oci-containers = {
-        forceSSL = true;
+      backend = "podman";
-        useACMEHost = "ha.ppp.pm";
 
-        extraConfig = ''
+      containers.homeassistant = {
-          proxy_buffering off;
+        image = "ghcr.io/home-assistant/home-assistant:stable";
-        '';
 
-        locations."/" = {
+        volumes = [
-          proxyPass = "http://127.0.0.1:8123";
+          "/home/alex/.config/home-assistant:/config"
-          proxyWebsockets = true;
+          "${configFile}:/config/configuration.yaml:ro"
-        };
+          # Pass in bluetooth
+          "/run/dbus:/run/dbus:ro"
+        ];
+
+        environment.TZ = "Europe/Stockholm";
+
+        extraOptions = [
+          "--network=host"
+
+          # Allows HA to perform low-level network operations (scan/reset adapter)
+          "--cap-add=NET_ADMIN"
+          "--cap-add=NET_RAW"
+
+          # Pass in Zigbee antenna
+          "--device=/dev/serial/by-id/usb-Nabu_Casa_ZBT-2_9C139EAAD464-if00:/dev/ttyACM0"
+        ];
       };
     };
 
-    # Trigger reset via udev when hci0 disappears
-    udev.extraRules = ''
-      ACTION=="remove", SUBSYSTEM=="bluetooth", KERNEL=="hci0", \
-        TAG+="systemd", ENV{SYSTEMD_WANTS}+="bt-reset.service"
-    '';
-  };
-
-  systemd = {
     services = {
-      # Trigger reset on bluetoothd failure
+      blueman.enable = true;
-      bluetooth = {
-        unitConfig.OnFailure = [ "bt-reset.service" ];
-      };
 
-      bt-reset = {
+      nginx = lib.mkIf nginxEnabled {
-        description = "Reset Bluetooth adapter";
+        recommendedProxySettings = true;
-        after = [ "bluetooth.service" ];
 
-        serviceConfig = {
+        virtualHosts."ha.ppp.pm" = {
-          Type = "oneshot";
+          forceSSL = true;
-          ExecStart = script;
+          useACMEHost = "ha.ppp.pm";
 
-          Restart = "on-failure";
+          extraConfig = ''
-          RestartSec = "10s";
+            proxy_buffering off;
-          StartLimitIntervalSec = "120";
+          '';
-          StartLimitBurst = 3;
-        };
-      };
-    };
 
-    timers.bt-reset = {
+          locations."/" = {
-      description = "Periodically reset Bluetooth adapter";
+            proxyPass = "http://127.0.0.1:8123";
-      wantedBy = [ "timers.target" ];
+            proxyWebsockets = true;
-      timerConfig = {
-        OnBootSec = "5min"; # first run 5 min after boot
-        OnUnitActiveSec = "4h"; # then every 4 hours
-        RandomizedDelaySec = "5min";
-      };
-    };
-
-    user = {
-      timers = {
-        "update-hetzner-dns" = {
-          unitConfig = {
-            Description = "updates Hetzner DNS records";
           };
-
-          timerConfig = {
-            Unit = "update-hetzner-dns.service";
-            OnCalendar = "*-*-* *:00/30:00";
-            Persistent = true;
-          };
-
-          wantedBy = [ "timers.target" ];
         };
       };
 
+      # Trigger reset via udev when hci0 disappears
+      udev.extraRules = ''
+        ACTION=="remove", SUBSYSTEM=="bluetooth", KERNEL=="hci0", \
+          TAG+="systemd", ENV{SYSTEMD_WANTS}+="bt-reset.service"
+      '';
+    };
+
+    systemd = {
       services = {
-        "update-hetzner-dns" = {
+        # Trigger reset on bluetoothd failure
-          unitConfig = {
+        bluetooth = {
-            Description = "updates Hetzner DNS records";
+          unitConfig.OnFailure = [ "bt-reset.service" ];
-          };
+        };
+
+        bt-reset = {
+          description = "Reset Bluetooth adapter";
+          after = [ "bluetooth.service" ];
 
           serviceConfig = {
-            Type = "exec";
+            Type = "oneshot";
-            EnvironmentFile = config.age.secrets.hetzner-dns.path;
+            ExecStart = btResetScript;
+
+            Restart = "on-failure";
+            RestartSec = "10s";
+            StartLimitIntervalSec = "120";
+            StartLimitBurst = 3;
           };
+        };
+      };
 
-          path = [
+      timers.bt-reset = {
-            pkgs.curl
+        description = "Periodically reset Bluetooth adapter";
-            pkgs.coreutils
+        wantedBy = [ "timers.target" ];
-            pkgs.jq
+        timerConfig = {
-          ];
+          OnBootSec = "5min"; # first run 5 min after boot
+          OnUnitActiveSec = "4h"; # then every 4 hours
+          RandomizedDelaySec = "5min";
+        };
+      };
 
-          script = ''
+      user = {
-            SUBDOMAINS="ha komga romm"
+        timers = {
-            INTERFACE="enp3s0"
+          "update-hetzner-dns" = {
+            unitConfig = {
+              Description = "updates Hetzner DNS records";
+            };
 
-            CURRENT_IP=$(curl -s --fail --interface "$INTERFACE" ifconfig.me)
+            timerConfig = {
+              Unit = "update-hetzner-dns.service";
+              OnCalendar = "*-*-* *:00/30:00";
+              Persistent = true;
+            };
 
-            for SUBDOMAIN in $SUBDOMAINS; do
+            wantedBy = [ "timers.target" ];
-              LAST_IP_FILE="/tmp/hetzner-dns-''${SUBDOMAIN}-ip"
+          };
+        };
 
-              LAST_IP=""
+        services = {
-              if [[ -f "$LAST_IP_FILE" ]]; then
+          "update-hetzner-dns" = {
-                  LAST_IP=$(cat "$LAST_IP_FILE")
+            unitConfig = {
-              fi
+              Description = "updates Hetzner DNS records";
+            };
 
-              if [[ "$CURRENT_IP" == "$LAST_IP" ]]; then
+            serviceConfig = {
-                  echo "$SUBDOMAIN: IP unchanged, NOOP update."
+              Type = "exec";
-              else
+              EnvironmentFile = config.age.secrets.hetzner-dns.path;
-                  echo "$SUBDOMAIN: Updating IP"
+            };
 
-                  JSON_BODY=$(jq -n --arg ip "$CURRENT_IP" '{records: [{value: $ip}]}')
+            path = [
+              pkgs.curl
+              pkgs.coreutils
+              pkgs.jq
+            ];
 
-                  curl \
+            script = ''
-                    --fail \
+              SUBDOMAINS="ha komga romm"
-                  	-X POST \
+              INTERFACE="enp3s0"
-                    -H "Authorization: Bearer $HETZNER_API_TOKEN" \
+
-                    -H "Content-Type: application/json" \
+              CURRENT_IP=$(curl -s --fail --interface "$INTERFACE" ifconfig.me)
-                    -d "$JSON_BODY" \
+
-                    "https://api.hetzner.cloud/v1/zones/ppp.pm/rrsets/''${SUBDOMAIN}/A/actions/set_records" \
+              for SUBDOMAIN in $SUBDOMAINS; do
-                    && echo $CURRENT_IP > $LAST_IP_FILE
+                LAST_IP_FILE="/tmp/hetzner-dns-''${SUBDOMAIN}-ip"
-              fi
+
-            done
+                LAST_IP=""
-          '';
+                if [[ -f "$LAST_IP_FILE" ]]; then
+                    LAST_IP=$(cat "$LAST_IP_FILE")
+                fi
+
+                if [[ "$CURRENT_IP" == "$LAST_IP" ]]; then
+                    echo "$SUBDOMAIN: IP unchanged, NOOP update."
+                else
+                    echo "$SUBDOMAIN: Updating IP"
+
+                    JSON_BODY=$(jq -n --arg ip "$CURRENT_IP" '{records: [{value: $ip}]}')
+
+                    curl \
+                      --fail \
+                    	-X POST \
+                      -H "Authorization: Bearer $HETZNER_API_TOKEN" \
+                      -H "Content-Type: application/json" \
+                      -d "$JSON_BODY" \
+                      "https://api.hetzner.cloud/v1/zones/ppp.pm/rrsets/''${SUBDOMAIN}/A/actions/set_records" \
+                      && echo $CURRENT_IP > $LAST_IP_FILE
+                fi
+              done
+            '';
+          };
         };
       };
     };
-  };
 
-  age = {
+    age = {
-    secrets = {
+      secrets = {
-      "hetzner-dns" = {
+        "hetzner-dns" = {
-        file = ../../../../secrets/manatee/hetzner-dns.age;
+          file = ../../../../secrets/manatee/hetzner-dns.age;
-        owner = "alex";
+          owner = "alex";
-        group = "users";
+          group = "users";
+        };
       };
     };
   };