diff options
Diffstat (limited to 'dashboards')
| -rw-r--r-- | dashboards/gen_smart.py | 53 | ||||
| -rw-r--r-- | dashboards/gen_zfs.py | 5 | ||||
| -rw-r--r-- | dashboards/panels.py | 13 |
3 files changed, 50 insertions, 21 deletions
diff --git a/dashboards/gen_smart.py b/dashboards/gen_smart.py index de9ab34..76ca8fe 100644 --- a/dashboards/gen_smart.py +++ b/dashboards/gen_smart.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Generate the "Drive Health (SMART)" Grafana dashboard. -Run this and commit the result: +modules/dashboards.nix runs this at build time; the JSON is not committed. To see what +it produces: - python3 dashboards/gen_smart.py > dashboards/smart.json + python3 dashboards/gen_smart.py | jq . A CrystalDiskInfo-style view: a Good/Caution/Bad verdict per drive, the full raw SMART attribute table, and self-test history. @@ -63,12 +64,27 @@ HOST = 'host=~"$host"' ATA = f'{HOST}, device=~"$ata_device"' SAS = f'{HOST}, device=~"$sas_device"' -# Splits "sde · GOOD · 6.0 TB · SAS 7.2K · HGST HUS726060AL5210 · NAHPJS45" into a -# selectable value (the leading device name) and display text (the whole line). The -# groups nest rather than sit side by side, which is what lets text be a superset of -# value; matching on whitespace keeps it independent of the separator the collector -# chose. -ROW_REGEX = r"/^(?<text>(?<value>\S+)\s.*)$/" +# The per-drive sections come from smartctl:row, one preformatted line per drive. +# +# It has to be query_result() rather than the more obvious +# label_values(smartctl:row{...}, row), because label_values() asks Prometheus for every +# value of `row` seen anywhere in the dashboard's time range -- 24 hours by default. +# Replace a drive and that window still holds the line the *old* drive produced. Both +# lines then reduce to the same variable value (the device name, which the panels below +# select on), and Grafana's uniqBy(options, 'value') keeps whichever came back first. +# The result is one row headed with the removed drive's model and serial, no row for its +# replacement, and panels underneath showing the new drive's numbers. query_result() is +# an instant query at the end of the range, so it can only return drives that are in the +# machine now. +DEVICE_QUERY = 'query_result(smartctl:row{{bus="{bus}", host=~"$host"}})' + +# Pulls the value out of a query_result() line, which Grafana renders as +# smartctl:row{bus="sas",device="sde",...,row="sde · GOOD · 6.0 TB · ...",...} 0 17883… +# Anchoring on `,row="` rather than `row="` keeps it from matching a `row` that turned up +# inside some other label's text. The groups nest rather than sit side by side, which is +# what lets `text` (the whole line, shown in the row title) be a superset of `value` (the +# bare device name, which every device=~"$..." selector below substitutes). +ROW_REGEX = r'/,row="(?<text>(?<value>[^\s"]+)[^"]*)"/' def power_on(sel=""): @@ -579,19 +595,20 @@ print(json.dumps(dashboard( "error counters for SAS, and self-test history.", panels=panels, variables=[ - # smartctl:row is one rendered line per drive, "sde · GOOD · 6.0 TB · ...". - # The nested capture groups keep the *value* a bare device name, so every - # `device=~"$ata_device"` selector below is unaffected, while the *text* the UI - # shows -- and that ${ata_device:text} puts in the row title -- is the whole - # line. That is what makes a collapsed row identifiable without opening it. - # One machine at a time. Not multi-select and no "All" -- see the comment - # on HOST above for why selecting two hosts at once would silently merge - # same-named drives rather than showing both. + # One machine at a time. Not multi-select and no "All" -- see the comment on + # HOST above for why selecting two hosts at once would silently merge same-named + # drives rather than showing both. This one stays on label_values deliberately: + # its window being the dashboard's time range is what keeps a machine that is + # currently down still selectable, which is exactly when you want to look at it. query_var("host", "label_values(smartctl_device_smart_status, host)", label="Host", multi=False, include_all=False), - query_var("ata_device", 'label_values(smartctl:row{bus="ata", host=~"$host"}, row)', + # smartctl:row is one rendered line per drive, "sde · GOOD · 6.0 TB · ...", and + # ${ata_device:text} puts that whole line in the repeated row's title. That is + # what makes a collapsed row identifiable without opening it. See DEVICE_QUERY + # and ROW_REGEX above for why these are instant queries. + query_var("ata_device", DEVICE_QUERY.format(bus="ata"), label="SATA drive", regex=ROW_REGEX), - query_var("sas_device", 'label_values(smartctl:row{bus="sas", host=~"$host"}, row)', + query_var("sas_device", DEVICE_QUERY.format(bus="sas"), label="SAS drive", regex=ROW_REGEX), ], tags=["smart", "storage", "hardware"], diff --git a/dashboards/gen_zfs.py b/dashboards/gen_zfs.py index 0057506..e00d03f 100644 --- a/dashboards/gen_zfs.py +++ b/dashboards/gen_zfs.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Generate the "ZFS Pools" Grafana dashboard. -Run this and commit the result: +modules/dashboards.nix runs this at build time; the JSON is not committed. To see what +it produces: - python3 dashboards/gen_zfs.py > dashboards/zfs-pools.json + python3 dashboards/gen_zfs.py | jq . Hand-writing ~2000 lines of Grafana panel JSON invites subtle inconsistencies, so panels are built from a handful of helpers instead -- every timeseries goes through one diff --git a/dashboards/panels.py b/dashboards/panels.py index faa60f7..919cb3c 100644 --- a/dashboards/panels.py +++ b/dashboards/panels.py @@ -237,6 +237,12 @@ def row(title, y, collapsed=False, children=None, repeat=None): return r +# Which editor tab Grafana opens the variable on. Only the UI reads this: at query time +# PrometheusVariableSupport.query() takes the `query` string and hands it straight to +# PrometheusMetricFindQuery, which re-parses it. Derive it so the two cannot disagree. +QRY_TYPES = {"label_names(": 0, "label_values(": 1, "metrics(": 2, "query_result(": 3} + + def query_var(name, query, label=None, multi=True, include_all=True, all_value=".*", current_all=True, hide=0, regex=None): """A Prometheus-backed template variable. @@ -244,13 +250,18 @@ def query_var(name, query, label=None, multi=True, include_all=True, all_value=" `regex` filters and rewrites the returned strings. Named capture groups `value` and `text` split one string into what queries substitute and what the UI shows -- nest them (`(?<text>(?<value>\\S+) .*)`) to show a superset of what you select on. + + `label_values(...)` resolves against /api/v1/label/<l>/values over the *dashboard's + time range*, so it lists every value seen in that window, not the ones that exist + now. `query_result(...)` is an instant query at the range's end instead. Prefer it + whenever a stale option would be actively wrong rather than merely extra. """ v = { "name": name, "label": label or name, "type": "query", "datasource": DS, - "query": {"qryType": 1, "query": query, + "query": {"qryType": QRY_TYPES.get(query.split("(")[0] + "(", 4), "query": query, "refId": "PrometheusVariableQueryEditor-VariableQuery"}, "refresh": 1, "includeAll": include_all, |