split into a flake with separable collectors/server modules

Replaces the single lgtm.nix with a flake exposing two NixOS modules under
services.telemetry, so the web services can move into a container declared in
the consuming config rather than one hardcoded here.

  modules/collectors.nix  host-side: node + smartctl exporters, the ZFS and
                          SMART textfile collector timers, Alloy journal
                          shipping. Must run on the host -- reads /proc, /sys,
                          raw block devices and ZFS ioctls.
  modules/server.nix      Prometheus, Loki, Grafana, optional Caddy.
  modules/common.nix      ports, shared by both halves.
  modules/dashboards.nix  jq fixup and grafana.com fetches.
  modules/packages.nix    the two collectors as ordinary packages.

Modules are exported as paths, not functions, so they evaluate with the
consumer's nixpkgs; the flake's own input only affects packages/checks/devShells.

Multi-host support. collectorHosts is a list, both scrape jobs relabel
__address__ into a `host` label, and all 133 queries in the two generated
dashboards are scoped to host=~"$host" behind a single-select Host picker.
`instance` could not serve as that key: one machine is scraped twice on
different ports (textfile metrics on :9100, smartctl_* on :9633) and the
recording rules join across that boundary.

Grafana's secret key becomes a plain secretKeyFile path wired through
LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No
sops-nix input.

nix flake check runs promtool over the rules, diffs the committed dashboard
JSON against what the generators produce, and evaluates both modules.

Also fixes smart-selftest emitting a placeholder identity line when smartctl
cannot open a device. smartctl exits 0 with well-formed but empty JSON in that
case, so "SSD - unknown model - no serial" was reaching a dashboard row title;
it now emits nothing and lets the smartctl:ident fallback apply.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ

1 files changed, 19 insertions, 19 deletions

diff --git a/rules/smartctl.yml b/rules/smartctl.yml
index dafed8d..3982e06 100644
--- a/rules/smartctl.yml
+++ b/rules/smartctl.yml
@@ -5,7 +5,7 @@
 #
 # The fleet is mixed: SATA drives expose the classic ATA attribute table, SAS drives
 # expose none of it and report health through SCSI error counters instead. Every rule
-# below is therefore made *total* -- it ends in `or (count by (device) (...) * 0)` so
+# below is therefore made *total* -- it ends in `or (count by (host, device) (...) * 0)` so
 # that each series exists with value 0 for every drive, including drives of the other
 # kind. Without that, the arithmetic in smartctl:health_code would silently drop drives
 # whose label sets did not match on both sides.
@@ -20,12 +20,12 @@ groups:
       # fails", so those are excluded rather than treated as an instant failure.
       - record: smartctl:attr_failing:count
         expr: |
-          count by (device) (
+          count by (host, device) (
             smartctl_device_attribute{attribute_value_type="value"}
-            <= on(device, attribute_id)
+            <= on(host, device, attribute_id)
             (smartctl_device_attribute{attribute_value_type="thresh"} > 0)
           )
-          or (count by (device) (smartctl_device_smart_status) * 0)
+          or (count by (host, device) (smartctl_device_smart_status) * 0)
 
       # Attributes whose raw value is a genuine warning sign across vendors:
       #   5   Reallocated_Sector_Ct     187 Reported_Uncorrect
@@ -35,13 +35,13 @@ groups:
       # values on Seagate drives by design and would flag healthy disks.
       - record: smartctl:attr_caution:count
         expr: |
-          count by (device) (
+          count by (host, device) (
             smartctl_device_attribute{
               attribute_value_type="raw",
               attribute_id=~"5|187|188|197|198|199"
             } > 0
           )
-          or (count by (device) (smartctl_device_smart_status) * 0)
+          or (count by (host, device) (smartctl_device_smart_status) * 0)
 
       # --- SAS / SCSI -----------------------------------------------------------
       # Grown defect list: sectors reallocated since the drive left the factory. The
@@ -49,15 +49,15 @@ groups:
       # acting on.
       - record: smartctl:scsi_defects:count
         expr: |
-          sum by (device) (smartctl_scsi_grown_defect_list)
-          or (count by (device) (smartctl_device_smart_status) * 0)
+          sum by (host, device) (smartctl_scsi_grown_defect_list)
+          or (count by (host, device) (smartctl_device_smart_status) * 0)
 
       # Errors the drive could not correct at all, which means data was lost.
       - record: smartctl:scsi_uncorrected:count
         expr: |
-          sum by (device) (smartctl_read_total_uncorrected_errors)
-          + sum by (device) (smartctl_write_total_uncorrected_errors)
-          or (count by (device) (smartctl_device_smart_status) * 0)
+          sum by (host, device) (smartctl_read_total_uncorrected_errors)
+          + sum by (host, device) (smartctl_write_total_uncorrected_errors)
+          or (count by (host, device) (smartctl_device_smart_status) * 0)
 
       # --- Verdict --------------------------------------------------------------
       # 0 good, 1 caution, 2 bad. The inner clamp_max on each group keeps two
@@ -66,7 +66,7 @@ groups:
         expr: |
           clamp_max(
             2 * clamp_max(
-                  (max by (device) (smartctl_device_smart_status) == bool 0)
+                  (max by (host, device) (smartctl_device_smart_status) == bool 0)
                 + (smartctl:attr_failing:count > bool 0)
                 + (smartctl:scsi_uncorrected:count > bool 0)
               , 1)
@@ -82,9 +82,9 @@ groups:
       # table, and only SCSI drives report a grown defect list.
       - record: smartctl:bus
         expr: |
-          label_replace(count by (device) (smartctl_device_attribute), "bus", "ata", "", "")
+          label_replace(count by (host, device) (smartctl_device_attribute), "bus", "ata", "", "")
           or
-          label_replace(count by (device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "")
+          label_replace(count by (host, device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "")
 
       # The identity string from collectors/smart-selftest.py, made total so a drive
       # still gets a row if that collector has not written its file yet. `unless` (not
@@ -92,11 +92,11 @@ groups:
       # `or` would emit two series per drive rather than deduplicating.
       - record: smartctl:ident
         expr: |
-          max by (device, label) (smart_drive_label)
+          max by (host, device, label) (smart_drive_label)
           or
           label_replace(
-            count by (device) (smartctl_device_smart_status)
-            unless on(device) smart_drive_label,
+            count by (host, device) (smartctl_device_smart_status)
+            unless on(host, device) smart_drive_label,
             "label", "identity pending", "", "")
 
       # One preformatted line per drive: "sde · GOOD · 6.0 TB · SAS 7.2K · model · SN".
@@ -111,6 +111,6 @@ groups:
              or label_replace(smartctl:health_code == 1, "verdict", "CAUTION", "", "")
              or label_replace(smartctl:health_code == 2, "verdict", "BAD", "", "")
             )
-            * on(device) group_left(bus) smartctl:bus
-            * on(device) group_left(label) smartctl:ident,
+            * on(host, device) group_left(bus) smartctl:bus
+            * on(host, device) group_left(label) smartctl:ident,
             "row", " · ", "device", "verdict", "label")