From a74c2877571afbfd04584b8821c754e9050b9e72 Mon Sep 17 00:00:00 2001 From: Tim Keller Date: Wed, 2 Sep 2026 11:22:36 -0500 Subject: fix drive rows surviving a drive replacement The repeated per-drive sections are driven by $ata_device/$sas_device, which were label_values(smartctl:row{...}, row). label_values resolves against /api/v1/label//values over the *dashboard's* time range -- 24h by default -- so it returns every row line seen in that window, not the ones that exist now. Replace a drive and the window still holds the line the old drive produced. Both lines carry the same variable value, because the regex reduces them to the bare device name that the panels select on, and Grafana's metricNamesToVariableValues ends in uniqBy(options, 'value'), which keeps whichever series Prometheus returned first. The dashboard then renders one row headed with the removed drive's model and serial, no row for its replacement, and panels underneath showing the new drive's numbers. query_result() is the only variable form the Prometheus datasource evaluates as an instant query (metric_find_query.ts: queryResultQuery hits /api/v1/query at range.to), so switch both device variables to it and re-anchor the regex on the row= label in the line Grafana renders for a vector result. Verified against a real Prometheus loading rules/smartctl.yml, with a fixture that replaces a SAS drive and a SATA drive in place: the old form headed sde with OLDBAD99 and sdz with the 870 EVO it no longer contains, the new form lists exactly the drives in the machine. The host variable deliberately stays on label_values -- its window covering the time range is what keeps an unreachable machine selectable. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ --- dashboards/gen_smart.py | 53 ++++++++++++++++++++++++++++++++----------------- dashboards/gen_zfs.py | 5 +++-- dashboards/panels.py | 13 +++++++++++- 3 files changed, 50 insertions(+), 21 deletions(-) (limited to 'dashboards') diff --git a/dashboards/gen_smart.py b/dashboards/gen_smart.py index de9ab34..76ca8fe 100644 --- a/dashboards/gen_smart.py +++ b/dashboards/gen_smart.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Generate the "Drive Health (SMART)" Grafana dashboard. -Run this and commit the result: +modules/dashboards.nix runs this at build time; the JSON is not committed. To see what +it produces: - python3 dashboards/gen_smart.py > dashboards/smart.json + python3 dashboards/gen_smart.py | jq . A CrystalDiskInfo-style view: a Good/Caution/Bad verdict per drive, the full raw SMART attribute table, and self-test history. @@ -63,12 +64,27 @@ HOST = 'host=~"$host"' ATA = f'{HOST}, device=~"$ata_device"' SAS = f'{HOST}, device=~"$sas_device"' -# Splits "sde · GOOD · 6.0 TB · SAS 7.2K · HGST HUS726060AL5210 · NAHPJS45" into a -# selectable value (the leading device name) and display text (the whole line). The -# groups nest rather than sit side by side, which is what lets text be a superset of -# value; matching on whitespace keeps it independent of the separator the collector -# chose. -ROW_REGEX = r"/^(?(?\S+)\s.*)$/" +# The per-drive sections come from smartctl:row, one preformatted line per drive. +# +# It has to be query_result() rather than the more obvious +# label_values(smartctl:row{...}, row), because label_values() asks Prometheus for every +# value of `row` seen anywhere in the dashboard's time range -- 24 hours by default. +# Replace a drive and that window still holds the line the *old* drive produced. Both +# lines then reduce to the same variable value (the device name, which the panels below +# select on), and Grafana's uniqBy(options, 'value') keeps whichever came back first. +# The result is one row headed with the removed drive's model and serial, no row for its +# replacement, and panels underneath showing the new drive's numbers. query_result() is +# an instant query at the end of the range, so it can only return drives that are in the +# machine now. +DEVICE_QUERY = 'query_result(smartctl:row{{bus="{bus}", host=~"$host"}})' + +# Pulls the value out of a query_result() line, which Grafana renders as +# smartctl:row{bus="sas",device="sde",...,row="sde · GOOD · 6.0 TB · ...",...} 0 17883… +# Anchoring on `,row="` rather than `row="` keeps it from matching a `row` that turned up +# inside some other label's text. The groups nest rather than sit side by side, which is +# what lets `text` (the whole line, shown in the row title) be a superset of `value` (the +# bare device name, which every device=~"$..." selector below substitutes). +ROW_REGEX = r'/,row="(?(?[^\s"]+)[^"]*)"/' def power_on(sel=""): @@ -579,19 +595,20 @@ print(json.dumps(dashboard( "error counters for SAS, and self-test history.", panels=panels, variables=[ - # smartctl:row is one rendered line per drive, "sde · GOOD · 6.0 TB · ...". - # The nested capture groups keep the *value* a bare device name, so every - # `device=~"$ata_device"` selector below is unaffected, while the *text* the UI - # shows -- and that ${ata_device:text} puts in the row title -- is the whole - # line. That is what makes a collapsed row identifiable without opening it. - # One machine at a time. Not multi-select and no "All" -- see the comment - # on HOST above for why selecting two hosts at once would silently merge - # same-named drives rather than showing both. + # One machine at a time. Not multi-select and no "All" -- see the comment on + # HOST above for why selecting two hosts at once would silently merge same-named + # drives rather than showing both. This one stays on label_values deliberately: + # its window being the dashboard's time range is what keeps a machine that is + # currently down still selectable, which is exactly when you want to look at it. query_var("host", "label_values(smartctl_device_smart_status, host)", label="Host", multi=False, include_all=False), - query_var("ata_device", 'label_values(smartctl:row{bus="ata", host=~"$host"}, row)', + # smartctl:row is one rendered line per drive, "sde · GOOD · 6.0 TB · ...", and + # ${ata_device:text} puts that whole line in the repeated row's title. That is + # what makes a collapsed row identifiable without opening it. See DEVICE_QUERY + # and ROW_REGEX above for why these are instant queries. + query_var("ata_device", DEVICE_QUERY.format(bus="ata"), label="SATA drive", regex=ROW_REGEX), - query_var("sas_device", 'label_values(smartctl:row{bus="sas", host=~"$host"}, row)', + query_var("sas_device", DEVICE_QUERY.format(bus="sas"), label="SAS drive", regex=ROW_REGEX), ], tags=["smart", "storage", "hardware"], diff --git a/dashboards/gen_zfs.py b/dashboards/gen_zfs.py index 0057506..e00d03f 100644 --- a/dashboards/gen_zfs.py +++ b/dashboards/gen_zfs.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Generate the "ZFS Pools" Grafana dashboard. -Run this and commit the result: +modules/dashboards.nix runs this at build time; the JSON is not committed. To see what +it produces: - python3 dashboards/gen_zfs.py > dashboards/zfs-pools.json + python3 dashboards/gen_zfs.py | jq . Hand-writing ~2000 lines of Grafana panel JSON invites subtle inconsistencies, so panels are built from a handful of helpers instead -- every timeseries goes through one diff --git a/dashboards/panels.py b/dashboards/panels.py index faa60f7..919cb3c 100644 --- a/dashboards/panels.py +++ b/dashboards/panels.py @@ -237,6 +237,12 @@ def row(title, y, collapsed=False, children=None, repeat=None): return r +# Which editor tab Grafana opens the variable on. Only the UI reads this: at query time +# PrometheusVariableSupport.query() takes the `query` string and hands it straight to +# PrometheusMetricFindQuery, which re-parses it. Derive it so the two cannot disagree. +QRY_TYPES = {"label_names(": 0, "label_values(": 1, "metrics(": 2, "query_result(": 3} + + def query_var(name, query, label=None, multi=True, include_all=True, all_value=".*", current_all=True, hide=0, regex=None): """A Prometheus-backed template variable. @@ -244,13 +250,18 @@ def query_var(name, query, label=None, multi=True, include_all=True, all_value=" `regex` filters and rewrites the returned strings. Named capture groups `value` and `text` split one string into what queries substitute and what the UI shows -- nest them (`(?(?\\S+) .*)`) to show a superset of what you select on. + + `label_values(...)` resolves against /api/v1/label//values over the *dashboard's + time range*, so it lists every value seen in that window, not the ones that exist + now. `query_result(...)` is an instant query at the range's end instead. Prefer it + whenever a stale option would be actively wrong rather than merely extra. """ v = { "name": name, "label": label or name, "type": "query", "datasource": DS, - "query": {"qryType": 1, "query": query, + "query": {"qryType": QRY_TYPES.get(query.split("(")[0] + "(", 4), "query": query, "refId": "PrometheusVariableQueryEditor-VariableQuery"}, "refresh": 1, "includeAll": include_all, -- cgit v1.3.1