From d2d1386abb358fa547a40fc8df613d799155b56d Mon Sep 17 00:00:00 2001 From: Tim Keller Date: Thu, 27 Aug 2026 12:45:44 -0500 Subject: split into a flake with separable collectors/server modules Replaces the single lgtm.nix with a flake exposing two NixOS modules under services.telemetry, so the web services can move into a container declared in the consuming config rather than one hardcoded here. modules/collectors.nix host-side: node + smartctl exporters, the ZFS and SMART textfile collector timers, Alloy journal shipping. Must run on the host -- reads /proc, /sys, raw block devices and ZFS ioctls. modules/server.nix Prometheus, Loki, Grafana, optional Caddy. modules/common.nix ports, shared by both halves. modules/dashboards.nix jq fixup and grafana.com fetches. modules/packages.nix the two collectors as ordinary packages. Modules are exported as paths, not functions, so they evaluate with the consumer's nixpkgs; the flake's own input only affects packages/checks/devShells. Multi-host support. collectorHosts is a list, both scrape jobs relabel __address__ into a `host` label, and all 133 queries in the two generated dashboards are scoped to host=~"$host" behind a single-select Host picker. `instance` could not serve as that key: one machine is scraped twice on different ports (textfile metrics on :9100, smartctl_* on :9633) and the recording rules join across that boundary. Grafana's secret key becomes a plain secretKeyFile path wired through LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No sops-nix input. nix flake check runs promtool over the rules, diffs the committed dashboard JSON against what the generators produce, and evaluates both modules. Also fixes smart-selftest emitting a placeholder identity line when smartctl cannot open a device. smartctl exits 0 with well-formed but empty JSON in that case, so "SSD - unknown model - no serial" was reaching a dashboard row title; it now emits nothing and lets the smartctl:ident fallback apply. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ --- rules/smartctl.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) (limited to 'rules') diff --git a/rules/smartctl.yml b/rules/smartctl.yml index dafed8d..3982e06 100644 --- a/rules/smartctl.yml +++ b/rules/smartctl.yml @@ -5,7 +5,7 @@ # # The fleet is mixed: SATA drives expose the classic ATA attribute table, SAS drives # expose none of it and report health through SCSI error counters instead. Every rule -# below is therefore made *total* -- it ends in `or (count by (device) (...) * 0)` so +# below is therefore made *total* -- it ends in `or (count by (host, device) (...) * 0)` so # that each series exists with value 0 for every drive, including drives of the other # kind. Without that, the arithmetic in smartctl:health_code would silently drop drives # whose label sets did not match on both sides. @@ -20,12 +20,12 @@ groups: # fails", so those are excluded rather than treated as an instant failure. - record: smartctl:attr_failing:count expr: | - count by (device) ( + count by (host, device) ( smartctl_device_attribute{attribute_value_type="value"} - <= on(device, attribute_id) + <= on(host, device, attribute_id) (smartctl_device_attribute{attribute_value_type="thresh"} > 0) ) - or (count by (device) (smartctl_device_smart_status) * 0) + or (count by (host, device) (smartctl_device_smart_status) * 0) # Attributes whose raw value is a genuine warning sign across vendors: # 5 Reallocated_Sector_Ct 187 Reported_Uncorrect @@ -35,13 +35,13 @@ groups: # values on Seagate drives by design and would flag healthy disks. - record: smartctl:attr_caution:count expr: | - count by (device) ( + count by (host, device) ( smartctl_device_attribute{ attribute_value_type="raw", attribute_id=~"5|187|188|197|198|199" } > 0 ) - or (count by (device) (smartctl_device_smart_status) * 0) + or (count by (host, device) (smartctl_device_smart_status) * 0) # --- SAS / SCSI ----------------------------------------------------------- # Grown defect list: sectors reallocated since the drive left the factory. The @@ -49,15 +49,15 @@ groups: # acting on. - record: smartctl:scsi_defects:count expr: | - sum by (device) (smartctl_scsi_grown_defect_list) - or (count by (device) (smartctl_device_smart_status) * 0) + sum by (host, device) (smartctl_scsi_grown_defect_list) + or (count by (host, device) (smartctl_device_smart_status) * 0) # Errors the drive could not correct at all, which means data was lost. - record: smartctl:scsi_uncorrected:count expr: | - sum by (device) (smartctl_read_total_uncorrected_errors) - + sum by (device) (smartctl_write_total_uncorrected_errors) - or (count by (device) (smartctl_device_smart_status) * 0) + sum by (host, device) (smartctl_read_total_uncorrected_errors) + + sum by (host, device) (smartctl_write_total_uncorrected_errors) + or (count by (host, device) (smartctl_device_smart_status) * 0) # --- Verdict -------------------------------------------------------------- # 0 good, 1 caution, 2 bad. The inner clamp_max on each group keeps two @@ -66,7 +66,7 @@ groups: expr: | clamp_max( 2 * clamp_max( - (max by (device) (smartctl_device_smart_status) == bool 0) + (max by (host, device) (smartctl_device_smart_status) == bool 0) + (smartctl:attr_failing:count > bool 0) + (smartctl:scsi_uncorrected:count > bool 0) , 1) @@ -82,9 +82,9 @@ groups: # table, and only SCSI drives report a grown defect list. - record: smartctl:bus expr: | - label_replace(count by (device) (smartctl_device_attribute), "bus", "ata", "", "") + label_replace(count by (host, device) (smartctl_device_attribute), "bus", "ata", "", "") or - label_replace(count by (device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "") + label_replace(count by (host, device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "") # The identity string from collectors/smart-selftest.py, made total so a drive # still gets a row if that collector has not written its file yet. `unless` (not @@ -92,11 +92,11 @@ groups: # `or` would emit two series per drive rather than deduplicating. - record: smartctl:ident expr: | - max by (device, label) (smart_drive_label) + max by (host, device, label) (smart_drive_label) or label_replace( - count by (device) (smartctl_device_smart_status) - unless on(device) smart_drive_label, + count by (host, device) (smartctl_device_smart_status) + unless on(host, device) smart_drive_label, "label", "identity pending", "", "") # One preformatted line per drive: "sde · GOOD · 6.0 TB · SAS 7.2K · model · SN". @@ -111,6 +111,6 @@ groups: or label_replace(smartctl:health_code == 1, "verdict", "CAUTION", "", "") or label_replace(smartctl:health_code == 2, "verdict", "BAD", "", "") ) - * on(device) group_left(bus) smartctl:bus - * on(device) group_left(label) smartctl:ident, + * on(host, device) group_left(bus) smartctl:bus + * on(host, device) group_left(label) smartctl:ident, "row", " · ", "device", "verdict", "label") -- cgit v1.3.1