From d2d1386abb358fa547a40fc8df613d799155b56d Mon Sep 17 00:00:00 2001 From: Tim Keller Date: Thu, 27 Aug 2026 12:45:44 -0500 Subject: split into a flake with separable collectors/server modules Replaces the single lgtm.nix with a flake exposing two NixOS modules under services.telemetry, so the web services can move into a container declared in the consuming config rather than one hardcoded here. modules/collectors.nix host-side: node + smartctl exporters, the ZFS and SMART textfile collector timers, Alloy journal shipping. Must run on the host -- reads /proc, /sys, raw block devices and ZFS ioctls. modules/server.nix Prometheus, Loki, Grafana, optional Caddy. modules/common.nix ports, shared by both halves. modules/dashboards.nix jq fixup and grafana.com fetches. modules/packages.nix the two collectors as ordinary packages. Modules are exported as paths, not functions, so they evaluate with the consumer's nixpkgs; the flake's own input only affects packages/checks/devShells. Multi-host support. collectorHosts is a list, both scrape jobs relabel __address__ into a `host` label, and all 133 queries in the two generated dashboards are scoped to host=~"$host" behind a single-select Host picker. `instance` could not serve as that key: one machine is scraped twice on different ports (textfile metrics on :9100, smartctl_* on :9633) and the recording rules join across that boundary. Grafana's secret key becomes a plain secretKeyFile path wired through LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No sops-nix input. nix flake check runs promtool over the rules, diffs the committed dashboard JSON against what the generators produce, and evaluates both modules. Also fixes smart-selftest emitting a placeholder identity line when smartctl cannot open a device. smartctl exits 0 with well-formed but empty JSON in that case, so "SSD - unknown model - no serial" was reaching a dashboard row title; it now emits nothing and lets the smartctl:ident fallback apply. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ --- dashboards/gen_zfs.py | 76 ++++++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 37 deletions(-) (limited to 'dashboards/gen_zfs.py') diff --git a/dashboards/gen_zfs.py b/dashboards/gen_zfs.py index 4edb38b..0057506 100644 --- a/dashboards/gen_zfs.py +++ b/dashboards/gen_zfs.py @@ -20,7 +20,8 @@ If you add a panel, add the metric to the collector first and keep this list hon import json -from panels import (DS, OK_BAD, bargauge, gauge, gp, row, stat, table, timeseries) +from panels import (DS, OK_BAD, bargauge, gauge, gp, query_var, row, stat, table, + timeseries) STATE_MAP = [{ @@ -46,7 +47,12 @@ BOOL_MAP = [{ }, }] -POOL = 'pool=~"$pool"' +# Every query on this dashboard is scoped to exactly one machine. The host variable is +# single-select with no "All" on purpose: pool names are not unique across a fleet, and +# the `max by (pool)` reductions and joinByField transformations below would silently +# merge two different pools that happen to share a name. +HOST = 'host=~"$host"' +POOL = f'{HOST}, pool=~"$pool"' panels = [] # ====================================================================================== @@ -124,7 +130,7 @@ panels.append(stat( panels.append(stat( "Snapshots Overdue", [(f'count((time() - zfs_snapshot_newest_timestamp_seconds{{{POOL}}}) > on(label) ' - f'group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)', "overdue")], + f'group_left() (2 * zfs_snapshot_interval_seconds{{{HOST}}})) or vector(0)', "overdue")], gp(15, 5, 4, 4), thresholds=OK_BAD, colormode="background", desc="Dataset/label pairs whose newest snapshot is more than two intervals old — " "i.e. the auto-snapshot timer has missed at least one run.", @@ -294,7 +300,7 @@ panels.append(bargauge( panels.append(bargauge( "Snapshots vs Retention Target", [(f'min by (label) (zfs_snapshot_count{{{POOL}}} / on(label) group_left() ' - f'zfs_snapshot_retention_target)', "{{label}}")], + f'zfs_snapshot_retention_target{{{HOST}}})', "{{label}}")], gp(8, 43, 8, 8), unit="percentunit", maxv=1.2, thresholds={"mode": "absolute", "steps": [ {"color": "red", "value": None}, @@ -329,15 +335,16 @@ panels.append(timeseries( panels.append(stat( "Snapshot / Scrub Job Failures", - [('sum(node_systemd_unit_state{name=~"zfs-(snapshot|scrub).*",state="failed"}) ' - 'or vector(0)', "failed units")], + [(f'sum(node_systemd_unit_state{{{HOST}, name=~"zfs-(snapshot|scrub).*", ' + f'state="failed"}}) or vector(0)', "failed units")], gp(0, 59, 6, 6), thresholds=OK_BAD, colormode="background", desc="systemd units for the auto-snapshot and auto-scrub timers currently in a " "failed state.", )) panels.append(bargauge( "Time Since Timer Last Fired", - [('time() - node_systemd_timer_last_trigger_seconds{name=~"zfs-.*"}', "{{name}}")], + [(f'time() - node_systemd_timer_last_trigger_seconds{{{HOST}, ' + f'name=~"zfs-.*"}}', "{{name}}")], gp(6, 59, 18, 6), unit="s", desc="When each zfs-* systemd timer last triggered. This is the job side of the " "picture; the snapshot ages above are the result side.", @@ -445,9 +452,9 @@ panels.append(row("Performance", 91)) panels.append(timeseries( "Dataset Throughput", [ - (f'rate(node_zfs_zpool_dataset_nread{{zpool=~"$pool"}}[$__rate_interval])', + (f'rate(node_zfs_zpool_dataset_nread{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} read"), - (f'-rate(node_zfs_zpool_dataset_nwritten{{zpool=~"$pool"}}[$__rate_interval])', + (f'-rate(node_zfs_zpool_dataset_nwritten{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} write"), ], gp(0, 92, 12, 8), unit="Bps", @@ -456,9 +463,9 @@ panels.append(timeseries( )) panels.append(timeseries( "Dataset IOPS", [ - (f'rate(node_zfs_zpool_dataset_reads{{zpool=~"$pool"}}[$__rate_interval])', + (f'rate(node_zfs_zpool_dataset_reads{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} read"), - (f'-rate(node_zfs_zpool_dataset_writes{{zpool=~"$pool"}}[$__rate_interval])', + (f'-rate(node_zfs_zpool_dataset_writes{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} write"), ], gp(12, 92, 12, 8), unit="iops", @@ -467,24 +474,24 @@ panels.append(timeseries( panels.append(timeseries( "ARC Size", [ - ("node_zfs_arc_size", "size"), - ("node_zfs_arc_c_max", "target max (c_max)"), - ("node_zfs_arc_c_min", "target min (c_min)"), + (f"node_zfs_arc_size{{{HOST}}}", "size"), + (f"node_zfs_arc_c_max{{{HOST}}}", "target max (c_max)"), + (f"node_zfs_arc_c_min{{{HOST}}}", "target min (c_min)"), ], gp(0, 100, 8, 8), unit="bytes", minval=0, )) panels.append(timeseries( "ARC Hit Ratio", - [("rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min(" - "rate(node_zfs_arc_hits[$__rate_interval]) + " - "rate(node_zfs_arc_misses[$__rate_interval]), 1)", "hit ratio")], + [(f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval]) / clamp_min(" + f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval]) + " + f"rate(node_zfs_arc_misses{{{HOST}}}[$__rate_interval]), 1)", "hit ratio")], gp(8, 100, 8, 8), unit="percentunit", minval=0, desc="clamp_min avoids a divide-by-zero while the pool is idle.", )) panels.append(timeseries( "ARC Hits / Misses", [ - ("rate(node_zfs_arc_hits[$__rate_interval])", "hits"), - ("rate(node_zfs_arc_misses[$__rate_interval])", "misses"), + (f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval])", "hits"), + (f"rate(node_zfs_arc_misses{{{HOST}}}[$__rate_interval])", "misses"), ], gp(16, 100, 8, 8), unit="ops", minval=0, )) @@ -553,7 +560,7 @@ panels.append(row("Pool Internals", 108, collapsed=True, children=[ panels.append(row("Collector Health", 109, collapsed=True, children=[ timeseries( "ZFS Collector Duration", - [("zfs_metrics_collection_duration_seconds", "collection")], + [(f"zfs_metrics_collection_duration_seconds{{{HOST}}}", "collection")], gp(0, 110, 8, 7), unit="s", minval=0, desc="How long collectors/zfs-metrics.py takes. Listing every snapshot is the " "slow part; if this approaches the 1 minute timer interval, raise " @@ -561,7 +568,7 @@ panels.append(row("Collector Health", 109, collapsed=True, children=[ ), timeseries( "Textfile Age", - [("time() - node_textfile_mtime_seconds", "{{file}}")], + [(f"time() - node_textfile_mtime_seconds{{{HOST}}}", "{{file}}")], gp(8, 110, 8, 7), unit="s", minval=0, desc="Age of each .prom file. If this climbs without bound the collector has " "stopped writing and every ZFS metric above is stale.", @@ -569,7 +576,7 @@ panels.append(row("Collector Health", 109, collapsed=True, children=[ timeseries( "Scrape Health", [ ('up{job="node"}', "node exporter up"), - ("node_textfile_scrape_error", "textfile parse error"), + (f"node_textfile_scrape_error{{{HOST}}}", "textfile parse error"), ], gp(16, 110, 8, 7), minval=0, desc="node_textfile_scrape_error goes to 1 if a .prom file is malformed, in " @@ -591,21 +598,16 @@ dashboard = { "graphTooltip": 1, "refresh": "1m", "time": {"from": "now-24h", "to": "now"}, - "templating": {"list": [{ - "name": "pool", - "label": "Pool", - "type": "query", - "datasource": DS, - "query": {"qryType": 1, "query": "label_values(zfs_pool_health_code, pool)", - "refId": "PrometheusVariableQueryEditor-VariableQuery"}, - "refresh": 1, - "includeAll": True, - "allValue": ".*", - "multi": True, - "current": {"text": ["All"], "value": ["$__all"]}, - "options": [], - "sort": 1, - }]}, + "templating": {"list": [ + # One machine at a time: not multi-select and no "All". See the comment on HOST + # above for why two hosts at once would silently merge same-named pools. + query_var("host", "label_values(zfs_pool_health_code, host)", + label="Host", multi=False, include_all=False), + # Pools are listed for the selected host only, so switching hosts re-populates + # the picker rather than offering pools that do not exist on this machine. + query_var("pool", 'label_values(zfs_pool_health_code{host=~"$host"}, pool)', + label="Pool"), + ]}, "annotations": {"list": [{ "builtIn": 1, "datasource": {"type": "datasource", "uid": "grafana"}, "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)", -- cgit v1.3.1