From d2d1386abb358fa547a40fc8df613d799155b56d Mon Sep 17 00:00:00 2001 From: Tim Keller Date: Thu, 27 Aug 2026 12:45:44 -0500 Subject: split into a flake with separable collectors/server modules Replaces the single lgtm.nix with a flake exposing two NixOS modules under services.telemetry, so the web services can move into a container declared in the consuming config rather than one hardcoded here. modules/collectors.nix host-side: node + smartctl exporters, the ZFS and SMART textfile collector timers, Alloy journal shipping. Must run on the host -- reads /proc, /sys, raw block devices and ZFS ioctls. modules/server.nix Prometheus, Loki, Grafana, optional Caddy. modules/common.nix ports, shared by both halves. modules/dashboards.nix jq fixup and grafana.com fetches. modules/packages.nix the two collectors as ordinary packages. Modules are exported as paths, not functions, so they evaluate with the consumer's nixpkgs; the flake's own input only affects packages/checks/devShells. Multi-host support. collectorHosts is a list, both scrape jobs relabel __address__ into a `host` label, and all 133 queries in the two generated dashboards are scoped to host=~"$host" behind a single-select Host picker. `instance` could not serve as that key: one machine is scraped twice on different ports (textfile metrics on :9100, smartctl_* on :9633) and the recording rules join across that boundary. Grafana's secret key becomes a plain secretKeyFile path wired through LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No sops-nix input. nix flake check runs promtool over the rules, diffs the committed dashboard JSON against what the generators produce, and evaluates both modules. Also fixes smart-selftest emitting a placeholder identity line when smartctl cannot open a device. smartctl exits 0 with well-formed but empty JSON in that case, so "SSD - unknown model - no serial" was reaching a dashboard row title; it now emits nothing and lets the smartctl:ident fallback apply. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ --- dashboards/gen_smart.py | 92 ++++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 39 deletions(-) (limited to 'dashboards/gen_smart.py') diff --git a/dashboards/gen_smart.py b/dashboards/gen_smart.py index 2cc0568..de9ab34 100644 --- a/dashboards/gen_smart.py +++ b/dashboards/gen_smart.py @@ -54,8 +54,14 @@ PASS_MAP = [{ }, }] -ATA = 'device=~"$ata_device"' -SAS = 'device=~"$sas_device"' +# Every query on this dashboard is scoped to exactly one machine. The host variable is +# deliberately single-select with no "All": `sda` is not a unique name across a fleet, +# and the `max by (device)` reductions and joinByField transformations below would +# silently merge two different physical drives into one row if more than one host were +# ever selected at a time. +HOST = 'host=~"$host"' +ATA = f'{HOST}, device=~"$ata_device"' +SAS = f'{HOST}, device=~"$sas_device"' # Splits "sde · GOOD · 6.0 TB · SAS 7.2K · HGST HUS726060AL5210 · NAHPJS45" into a # selectable value (the leading device name) and display text (the whole line). The @@ -80,7 +86,7 @@ def power_on(sel=""): source is larger is therefore never an over-estimate, and recovers a sane number on drives whose power-on counter under-counts or was reset. """ - braces = f"{{{sel}}}" if sel else "" + braces = "{" + ", ".join(x for x in (HOST, sel) if x) + "}" return (f'max by (device) (smartctl_device_power_on_seconds{braces} ' f'or (smart_selftest_last_lifetime_hours{braces} * 3600))') @@ -124,14 +130,14 @@ panels = [] panels.append(row("Fleet Overview", 0)) panels.append(stat( - "Healthy", [('count(smartctl:health_code == 0) or vector(0)', "drives")], + "Healthy", [(f'count(smartctl:health_code{{{HOST}}} == 0) or vector(0)', "drives")], gp(0, 1, 4, 4), colormode="background", thresholds={"mode": "absolute", "steps": [{"color": "green", "value": None}]}, desc="Drives with no failing attribute, no SCSI uncorrected errors and a passing " "SMART self-assessment.", )) panels.append(stat( - "Caution", [('count(smartctl:health_code == 1) or vector(0)', "drives")], + "Caution", [(f'count(smartctl:health_code{{{HOST}}} == 1) or vector(0)', "drives")], gp(4, 1, 4, 4), colormode="background", thresholds={"mode": "absolute", "steps": [ {"color": "green", "value": None}, {"color": "orange", "value": 1}]}, @@ -139,18 +145,19 @@ panels.append(stat( "defect list on SAS. Not failing yet — worth watching the trend.", )) panels.append(stat( - "Failing", [('count(smartctl:health_code == 2) or vector(0)', "drives")], + "Failing", [(f'count(smartctl:health_code{{{HOST}}} == 2) or vector(0)', "drives")], gp(8, 1, 4, 4), colormode="background", thresholds=OK_BAD, desc="SMART self-assessment failed, an attribute is at or below its threshold, or " "the drive reported errors it could not correct. Replace these.", )) panels.append(stat( - "Drives", [('count(smartctl_device_smart_status)', "total")], + "Drives", [(f'count(smartctl_device_smart_status{{{HOST}}})', "total")], gp(12, 1, 4, 4), )) panels.append(stat( "Hottest Drive", - [('max(smartctl_device_temperature{temperature_type="current"})', "max")], + [(f'max(smartctl_device_temperature{{{HOST}, temperature_type="current"}})', + "max")], gp(16, 1, 4, 4), unit="celsius", colormode="background", thresholds={"mode": "absolute", "steps": [ {"color": "green", "value": None}, @@ -167,12 +174,12 @@ panels.append(table( "All Drives", [ # group_left carries the descriptive labels from the info metric onto the # verdict, so this one query supplies device, model, serial and protocol. - ('smartctl:health_code * on(device) group_left(model_name, serial_number, ' - 'protocol) smartctl_device', None), - ('max by (device) (smartctl_device_temperature{temperature_type="current"})', - None), + (f'smartctl:health_code{{{HOST}}} * on(device) group_left(model_name, ' + f'serial_number, protocol) smartctl_device{{{HOST}}}', None), + (f'max by (device) (smartctl_device_temperature{{{HOST}, ' + f'temperature_type="current"}})', None), (power_on(), None), - ('max by (device) (smartctl_device_capacity_bytes)', None), + (f'max by (device) (smartctl_device_capacity_bytes{{{HOST}}})', None), ], gp(0, 5, 16, 10), keep="device|model_name|serial_number|protocol|Value.*", renames={ @@ -204,7 +211,8 @@ panels.append(table( panels.append(timeseries( "Temperatures", - [('smartctl_device_temperature{temperature_type="current"}', "{{device}}")], + [(f'smartctl_device_temperature{{{HOST}, temperature_type="current"}}', + "{{device}}")], gp(16, 5, 8, 10), unit="celsius", desc="Sustained operation above ~45C measurably shortens drive life.", )) @@ -418,22 +426,23 @@ panels.append(row("Self-Tests", 53)) panels.append(stat( "Drives Never Tested", - [('count(smartctl_device_smart_status) - ' - 'count(count by (device) (smart_selftest_last_lifetime_hours)) or ' - 'count(smartctl_device_smart_status)', "drives")], + [(f'count(smartctl_device_smart_status{{{HOST}}}) - ' + f'count(count by (device) (smart_selftest_last_lifetime_hours{{{HOST}}})) or ' + f'count(smartctl_device_smart_status{{{HOST}}})', "drives")], gp(0, 54, 4, 5), thresholds=OK_BAD, colormode="background", desc="Drives with no self-test log entries at all. smartd only runs tests if its " "config includes an -s schedule; merely enabling the service does not.", )) panels.append(stat( "Failed Self-Tests", - [('count(smart_selftest_last_passed == 0) or vector(0)', "tests")], + [(f'count(smart_selftest_last_passed{{{HOST}}} == 0) or vector(0)', "tests")], gp(4, 54, 4, 5), thresholds=OK_BAD, colormode="background", )) panels.append(stat( "Oldest Long Test", [(f'max({power_on()} / 3600 - on(device) group_right() ' - f'smart_selftest_last_lifetime_hours{{test_type="extended"}})', "hours ago")], + f'smart_selftest_last_lifetime_hours{{{HOST}, test_type="extended"}})', + "hours ago")], gp(8, 54, 4, 5), unit="h", thresholds={"mode": "absolute", "steps": [ {"color": "green", "value": None}, @@ -461,14 +470,14 @@ def _keyed(expr): panels.append(table( "Self-Test History", [ # Only this query keeps device and test_type as their own columns. - ('label_join(smart_selftest_last_passed, "drive_test", "/", "device", ' - '"test_type")', None), - (_keyed('smart_selftest_last_lifetime_hours'), None), - (_keyed('smart_selftest_count'), None), + (f'label_join(smart_selftest_last_passed{{{HOST}}}, "drive_test", "/", ' + f'"device", "test_type")', None), + (_keyed(f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None), + (_keyed(f'smart_selftest_count{{{HOST}}}'), None), # Powered-on hours elapsed since that test ran. group_right() keeps the # right-hand side's labels, so device and test_type survive the subtraction. (_keyed(power_on() + ' / 3600 - on(device) group_right() ' - 'smart_selftest_last_lifetime_hours'), None), + f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None), ], gp(12, 54, 12, 10), keep="drive_test|device|test_type|Value.*", renames={ @@ -501,22 +510,22 @@ panels.append(table( panels.append(row("Attribute Trends", 64, collapsed=True, children=[ timeseries( "Reallocated Sectors (all SATA drives)", - [('smartctl_device_attribute{attribute_id="5", attribute_value_type="raw"}', - "{{device}}")], + [(f'smartctl_device_attribute{{{HOST}, attribute_id="5", ' + f'attribute_value_type="raw"}}', "{{device}}")], gp(0, 65, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, ), timeseries( "Pending & Uncorrectable Sectors", - [('smartctl_device_attribute{attribute_id=~"197|198", ' - 'attribute_value_type="raw"}', "{{device}} {{attribute_name}}")], + [(f'smartctl_device_attribute{{{HOST}, attribute_id=~"197|198", ' + f'attribute_value_type="raw"}}', "{{device}} {{attribute_name}}")], gp(8, 65, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, ), timeseries( "CRC Errors (cabling)", - [('smartctl_device_attribute{attribute_id="199", attribute_value_type="raw"}', - "{{device}}")], + [(f'smartctl_device_attribute{{{HOST}, attribute_id="199", ' + f'attribute_value_type="raw"}}', "{{device}}")], gp(16, 65, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, desc="Attribute 199 counts errors on the SATA link rather than the platters. " @@ -524,14 +533,14 @@ panels.append(row("Attribute Trends", 64, collapsed=True, children=[ ), timeseries( "Data Written / Read (SAS)", [ - ('smartctl_device_bytes_written', "{{device}} written"), - ('smartctl_device_bytes_read', "{{device}} read"), + (f'smartctl_device_bytes_written{{{HOST}}}', "{{device}} written"), + (f'smartctl_device_bytes_read{{{HOST}}}', "{{device}} read"), ], gp(0, 72, 12, 7), unit="bytes", minval=0, ), timeseries( "Health Code History", - [('smartctl:health_code', "{{device}}")], + [(f'smartctl:health_code{{{HOST}}}', "{{device}}")], gp(12, 72, 12, 7), mappings=HEALTH_MAP, minval=0, maxval=2, custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"}, ), @@ -540,7 +549,7 @@ panels.append(row("Attribute Trends", 64, collapsed=True, children=[ panels.append(row("Collector Health", 65, collapsed=True, children=[ timeseries( "smartctl Exit Status", - [('smartctl_device_smartctl_exit_status', "{{device}}")], + [(f'smartctl_device_smartctl_exit_status{{{HOST}}}', "{{device}}")], gp(0, 66, 8, 7), minval=0, desc="Non-zero means smartctl itself had trouble reading the device. Bit 0 is " "a command-line problem, bit 1 a device-open failure, bit 3 a failing " @@ -548,13 +557,13 @@ panels.append(row("Collector Health", 65, collapsed=True, children=[ ), timeseries( "Self-Test Collector Duration", - [('smart_selftest_collection_duration_seconds', "collection")], + [(f'smart_selftest_collection_duration_seconds{{{HOST}}}', "collection")], gp(8, 66, 8, 7), unit="s", minval=0, ), timeseries( "Devices Seen", [ - ('smartctl_devices', "exporter"), - ('smart_selftest_devices', "self-test collector"), + (f'smartctl_devices{{{HOST}}}', "exporter"), + (f'smart_selftest_devices{{{HOST}}}', "self-test collector"), ], gp(16, 66, 8, 7), minval=0, desc="These should agree. A divergence means one of the two cannot see some " @@ -575,9 +584,14 @@ print(json.dumps(dashboard( # `device=~"$ata_device"` selector below is unaffected, while the *text* the UI # shows -- and that ${ata_device:text} puts in the row title -- is the whole # line. That is what makes a collapsed row identifiable without opening it. - query_var("ata_device", 'label_values(smartctl:row{bus="ata"}, row)', + # One machine at a time. Not multi-select and no "All" -- see the comment + # on HOST above for why selecting two hosts at once would silently merge + # same-named drives rather than showing both. + query_var("host", "label_values(smartctl_device_smart_status, host)", + label="Host", multi=False, include_all=False), + query_var("ata_device", 'label_values(smartctl:row{bus="ata", host=~"$host"}, row)', label="SATA drive", regex=ROW_REGEX), - query_var("sas_device", 'label_values(smartctl:row{bus="sas"}, row)', + query_var("sas_device", 'label_values(smartctl:row{bus="sas", host=~"$host"}, row)', label="SAS drive", regex=ROW_REGEX), ], tags=["smart", "storage", "hardware"], -- cgit v1.3.1