split into a flake with separable collectors/server modules

Replaces the single lgtm.nix with a flake exposing two NixOS modules under
services.telemetry, so the web services can move into a container declared in
the consuming config rather than one hardcoded here.

  modules/collectors.nix  host-side: node + smartctl exporters, the ZFS and
                          SMART textfile collector timers, Alloy journal
                          shipping. Must run on the host -- reads /proc, /sys,
                          raw block devices and ZFS ioctls.
  modules/server.nix      Prometheus, Loki, Grafana, optional Caddy.
  modules/common.nix      ports, shared by both halves.
  modules/dashboards.nix  jq fixup and grafana.com fetches.
  modules/packages.nix    the two collectors as ordinary packages.

Modules are exported as paths, not functions, so they evaluate with the
consumer's nixpkgs; the flake's own input only affects packages/checks/devShells.

Multi-host support. collectorHosts is a list, both scrape jobs relabel
__address__ into a `host` label, and all 133 queries in the two generated
dashboards are scoped to host=~"$host" behind a single-select Host picker.
`instance` could not serve as that key: one machine is scraped twice on
different ports (textfile metrics on :9100, smartctl_* on :9633) and the
recording rules join across that boundary.

Grafana's secret key becomes a plain secretKeyFile path wired through
LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No
sops-nix input.

nix flake check runs promtool over the rules, diffs the committed dashboard
JSON against what the generators produce, and evaluates both modules.

Also fixes smart-selftest emitting a placeholder identity line when smartctl
cannot open a device. smartctl exits 0 with well-formed but empty JSON in that
case, so "SSD - unknown model - no serial" was reaching a dashboard row title;
it now emits nothing and lets the smartctl:ident fallback apply.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ

1 files changed, 53 insertions, 39 deletions

diff --git a/dashboards/gen_smart.py b/dashboards/gen_smart.py
index 2cc0568..de9ab34 100644
--- a/dashboards/gen_smart.py
+++ b/dashboards/gen_smart.py
@@ -54,8 +54,14 @@ PASS_MAP = [{
     },
 }]
 
-ATA = 'device=~"$ata_device"'
-SAS = 'device=~"$sas_device"'
+# Every query on this dashboard is scoped to exactly one machine. The host variable is
+# deliberately single-select with no "All": `sda` is not a unique name across a fleet,
+# and the `max by (device)` reductions and joinByField transformations below would
+# silently merge two different physical drives into one row if more than one host were
+# ever selected at a time.
+HOST = 'host=~"$host"'
+ATA = f'{HOST}, device=~"$ata_device"'
+SAS = f'{HOST}, device=~"$sas_device"'
 
 # Splits "sde · GOOD · 6.0 TB · SAS 7.2K · HGST HUS726060AL5210 · NAHPJS45" into a
 # selectable value (the leading device name) and display text (the whole line). The
@@ -80,7 +86,7 @@ def power_on(sel=""):
     source is larger is therefore never an over-estimate, and recovers a sane number on
     drives whose power-on counter under-counts or was reset.
     """
-    braces = f"{{{sel}}}" if sel else ""
+    braces = "{" + ", ".join(x for x in (HOST, sel) if x) + "}"
     return (f'max by (device) (smartctl_device_power_on_seconds{braces} '
             f'or (smart_selftest_last_lifetime_hours{braces} * 3600))')
 
@@ -124,14 +130,14 @@ panels = []
 panels.append(row("Fleet Overview", 0))
 
 panels.append(stat(
-    "Healthy", [('count(smartctl:health_code == 0) or vector(0)', "drives")],
+    "Healthy", [(f'count(smartctl:health_code{{{HOST}}} == 0) or vector(0)', "drives")],
     gp(0, 1, 4, 4), colormode="background",
     thresholds={"mode": "absolute", "steps": [{"color": "green", "value": None}]},
     desc="Drives with no failing attribute, no SCSI uncorrected errors and a passing "
          "SMART self-assessment.",
 ))
 panels.append(stat(
-    "Caution", [('count(smartctl:health_code == 1) or vector(0)', "drives")],
+    "Caution", [(f'count(smartctl:health_code{{{HOST}}} == 1) or vector(0)', "drives")],
     gp(4, 1, 4, 4), colormode="background",
     thresholds={"mode": "absolute", "steps": [
         {"color": "green", "value": None}, {"color": "orange", "value": 1}]},
@@ -139,18 +145,19 @@ panels.append(stat(
          "defect list on SAS. Not failing yet — worth watching the trend.",
 ))
 panels.append(stat(
-    "Failing", [('count(smartctl:health_code == 2) or vector(0)', "drives")],
+    "Failing", [(f'count(smartctl:health_code{{{HOST}}} == 2) or vector(0)', "drives")],
     gp(8, 1, 4, 4), colormode="background", thresholds=OK_BAD,
     desc="SMART self-assessment failed, an attribute is at or below its threshold, or "
          "the drive reported errors it could not correct. Replace these.",
 ))
 panels.append(stat(
-    "Drives", [('count(smartctl_device_smart_status)', "total")],
+    "Drives", [(f'count(smartctl_device_smart_status{{{HOST}}})', "total")],
     gp(12, 1, 4, 4),
 ))
 panels.append(stat(
     "Hottest Drive",
-    [('max(smartctl_device_temperature{temperature_type="current"})', "max")],
+    [(f'max(smartctl_device_temperature{{{HOST}, temperature_type="current"}})',
+      "max")],
     gp(16, 1, 4, 4), unit="celsius", colormode="background",
     thresholds={"mode": "absolute", "steps": [
         {"color": "green", "value": None},
@@ -167,12 +174,12 @@ panels.append(table(
     "All Drives", [
         # group_left carries the descriptive labels from the info metric onto the
         # verdict, so this one query supplies device, model, serial and protocol.
-        ('smartctl:health_code * on(device) group_left(model_name, serial_number, '
-         'protocol) smartctl_device', None),
-        ('max by (device) (smartctl_device_temperature{temperature_type="current"})',
-         None),
+        (f'smartctl:health_code{{{HOST}}} * on(device) group_left(model_name, '
+         f'serial_number, protocol) smartctl_device{{{HOST}}}', None),
+        (f'max by (device) (smartctl_device_temperature{{{HOST}, '
+         f'temperature_type="current"}})', None),
         (power_on(), None),
-        ('max by (device) (smartctl_device_capacity_bytes)', None),
+        (f'max by (device) (smartctl_device_capacity_bytes{{{HOST}}})', None),
     ],
     gp(0, 5, 16, 10), keep="device|model_name|serial_number|protocol|Value.*",
     renames={
@@ -204,7 +211,8 @@ panels.append(table(
 
 panels.append(timeseries(
     "Temperatures",
-    [('smartctl_device_temperature{temperature_type="current"}', "{{device}}")],
+    [(f'smartctl_device_temperature{{{HOST}, temperature_type="current"}}',
+      "{{device}}")],
     gp(16, 5, 8, 10), unit="celsius",
     desc="Sustained operation above ~45C measurably shortens drive life.",
 ))
@@ -418,22 +426,23 @@ panels.append(row("Self-Tests", 53))
 
 panels.append(stat(
     "Drives Never Tested",
-    [('count(smartctl_device_smart_status) - '
-      'count(count by (device) (smart_selftest_last_lifetime_hours)) or '
-      'count(smartctl_device_smart_status)', "drives")],
+    [(f'count(smartctl_device_smart_status{{{HOST}}}) - '
+      f'count(count by (device) (smart_selftest_last_lifetime_hours{{{HOST}}})) or '
+      f'count(smartctl_device_smart_status{{{HOST}}})', "drives")],
     gp(0, 54, 4, 5), thresholds=OK_BAD, colormode="background",
     desc="Drives with no self-test log entries at all. smartd only runs tests if its "
          "config includes an -s schedule; merely enabling the service does not.",
 ))
 panels.append(stat(
     "Failed Self-Tests",
-    [('count(smart_selftest_last_passed == 0) or vector(0)', "tests")],
+    [(f'count(smart_selftest_last_passed{{{HOST}}} == 0) or vector(0)', "tests")],
     gp(4, 54, 4, 5), thresholds=OK_BAD, colormode="background",
 ))
 panels.append(stat(
     "Oldest Long Test",
     [(f'max({power_on()} / 3600 - on(device) group_right() '
-      f'smart_selftest_last_lifetime_hours{{test_type="extended"}})', "hours ago")],
+      f'smart_selftest_last_lifetime_hours{{{HOST}, test_type="extended"}})',
+      "hours ago")],
     gp(8, 54, 4, 5), unit="h",
     thresholds={"mode": "absolute", "steps": [
         {"color": "green", "value": None},
@@ -461,14 +470,14 @@ def _keyed(expr):
 panels.append(table(
     "Self-Test History", [
         # Only this query keeps device and test_type as their own columns.
-        ('label_join(smart_selftest_last_passed, "drive_test", "/", "device", '
-         '"test_type")', None),
-        (_keyed('smart_selftest_last_lifetime_hours'), None),
-        (_keyed('smart_selftest_count'), None),
+        (f'label_join(smart_selftest_last_passed{{{HOST}}}, "drive_test", "/", '
+         f'"device", "test_type")', None),
+        (_keyed(f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None),
+        (_keyed(f'smart_selftest_count{{{HOST}}}'), None),
         # Powered-on hours elapsed since that test ran. group_right() keeps the
         # right-hand side's labels, so device and test_type survive the subtraction.
         (_keyed(power_on() + ' / 3600 - on(device) group_right() '
-                'smart_selftest_last_lifetime_hours'), None),
+                f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None),
     ],
     gp(12, 54, 12, 10), keep="drive_test|device|test_type|Value.*",
     renames={
@@ -501,22 +510,22 @@ panels.append(table(
 panels.append(row("Attribute Trends", 64, collapsed=True, children=[
     timeseries(
         "Reallocated Sectors (all SATA drives)",
-        [('smartctl_device_attribute{attribute_id="5", attribute_value_type="raw"}',
-          "{{device}}")],
+        [(f'smartctl_device_attribute{{{HOST}, attribute_id="5", '
+          f'attribute_value_type="raw"}}', "{{device}}")],
         gp(0, 65, 8, 7), minval=0,
         custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
     ),
     timeseries(
         "Pending & Uncorrectable Sectors",
-        [('smartctl_device_attribute{attribute_id=~"197|198", '
-          'attribute_value_type="raw"}', "{{device}} {{attribute_name}}")],
+        [(f'smartctl_device_attribute{{{HOST}, attribute_id=~"197|198", '
+          f'attribute_value_type="raw"}}', "{{device}} {{attribute_name}}")],
         gp(8, 65, 8, 7), minval=0,
         custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
     ),
     timeseries(
         "CRC Errors (cabling)",
-        [('smartctl_device_attribute{attribute_id="199", attribute_value_type="raw"}',
-          "{{device}}")],
+        [(f'smartctl_device_attribute{{{HOST}, attribute_id="199", '
+          f'attribute_value_type="raw"}}', "{{device}}")],
         gp(16, 65, 8, 7), minval=0,
         custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
         desc="Attribute 199 counts errors on the SATA link rather than the platters. "
@@ -524,14 +533,14 @@ panels.append(row("Attribute Trends", 64, collapsed=True, children=[
     ),
     timeseries(
         "Data Written / Read (SAS)", [
-            ('smartctl_device_bytes_written', "{{device}} written"),
-            ('smartctl_device_bytes_read', "{{device}} read"),
+            (f'smartctl_device_bytes_written{{{HOST}}}', "{{device}} written"),
+            (f'smartctl_device_bytes_read{{{HOST}}}', "{{device}} read"),
         ],
         gp(0, 72, 12, 7), unit="bytes", minval=0,
     ),
     timeseries(
         "Health Code History",
-        [('smartctl:health_code', "{{device}}")],
+        [(f'smartctl:health_code{{{HOST}}}', "{{device}}")],
         gp(12, 72, 12, 7), mappings=HEALTH_MAP, minval=0, maxval=2,
         custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"},
     ),
@@ -540,7 +549,7 @@ panels.append(row("Attribute Trends", 64, collapsed=True, children=[
 panels.append(row("Collector Health", 65, collapsed=True, children=[
     timeseries(
         "smartctl Exit Status",
-        [('smartctl_device_smartctl_exit_status', "{{device}}")],
+        [(f'smartctl_device_smartctl_exit_status{{{HOST}}}', "{{device}}")],
         gp(0, 66, 8, 7), minval=0,
         desc="Non-zero means smartctl itself had trouble reading the device. Bit 0 is "
              "a command-line problem, bit 1 a device-open failure, bit 3 a failing "
@@ -548,13 +557,13 @@ panels.append(row("Collector Health", 65, collapsed=True, children=[
     ),
     timeseries(
         "Self-Test Collector Duration",
-        [('smart_selftest_collection_duration_seconds', "collection")],
+        [(f'smart_selftest_collection_duration_seconds{{{HOST}}}', "collection")],
         gp(8, 66, 8, 7), unit="s", minval=0,
     ),
     timeseries(
         "Devices Seen", [
-            ('smartctl_devices', "exporter"),
-            ('smart_selftest_devices', "self-test collector"),
+            (f'smartctl_devices{{{HOST}}}', "exporter"),
+            (f'smart_selftest_devices{{{HOST}}}', "self-test collector"),
         ],
         gp(16, 66, 8, 7), minval=0,
         desc="These should agree. A divergence means one of the two cannot see some "
@@ -575,9 +584,14 @@ print(json.dumps(dashboard(
         # `device=~"$ata_device"` selector below is unaffected, while the *text* the UI
         # shows -- and that ${ata_device:text} puts in the row title -- is the whole
         # line. That is what makes a collapsed row identifiable without opening it.
-        query_var("ata_device", 'label_values(smartctl:row{bus="ata"}, row)',
+        # One machine at a time. Not multi-select and no "All" -- see the comment
+        # on HOST above for why selecting two hosts at once would silently merge
+        # same-named drives rather than showing both.
+        query_var("host", "label_values(smartctl_device_smart_status, host)",
+                  label="Host", multi=False, include_all=False),
+        query_var("ata_device", 'label_values(smartctl:row{bus="ata", host=~"$host"}, row)',
                   label="SATA drive", regex=ROW_REGEX),
-        query_var("sas_device", 'label_values(smartctl:row{bus="sas"}, row)',
+        query_var("sas_device", 'label_values(smartctl:row{bus="sas", host=~"$host"}, row)',
                   label="SAS drive", regex=ROW_REGEX),
     ],
     tags=["smart", "storage", "hardware"],