From d2d1386abb358fa547a40fc8df613d799155b56d Mon Sep 17 00:00:00 2001 From: Tim Keller Date: Thu, 27 Aug 2026 12:45:44 -0500 Subject: split into a flake with separable collectors/server modules Replaces the single lgtm.nix with a flake exposing two NixOS modules under services.telemetry, so the web services can move into a container declared in the consuming config rather than one hardcoded here. modules/collectors.nix host-side: node + smartctl exporters, the ZFS and SMART textfile collector timers, Alloy journal shipping. Must run on the host -- reads /proc, /sys, raw block devices and ZFS ioctls. modules/server.nix Prometheus, Loki, Grafana, optional Caddy. modules/common.nix ports, shared by both halves. modules/dashboards.nix jq fixup and grafana.com fetches. modules/packages.nix the two collectors as ordinary packages. Modules are exported as paths, not functions, so they evaluate with the consumer's nixpkgs; the flake's own input only affects packages/checks/devShells. Multi-host support. collectorHosts is a list, both scrape jobs relabel __address__ into a `host` label, and all 133 queries in the two generated dashboards are scoped to host=~"$host" behind a single-select Host picker. `instance` could not serve as that key: one machine is scraped twice on different ports (textfile metrics on :9100, smartctl_* on :9633) and the recording rules join across that boundary. Grafana's secret key becomes a plain secretKeyFile path wired through LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No sops-nix input. nix flake check runs promtool over the rules, diffs the committed dashboard JSON against what the generators produce, and evaluates both modules. Also fixes smart-selftest emitting a placeholder identity line when smartctl cannot open a device. smartctl exits 0 with well-formed but empty JSON in that case, so "SSD - unknown model - no serial" was reaching a dashboard row title; it now emits nothing and lets the smartctl:ident fallback apply. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ --- dashboards/gen_smart.py | 92 +++++++++++++----------- dashboards/gen_zfs.py | 76 ++++++++++---------- dashboards/smart.json | 149 ++++++++++++++++++++++----------------- dashboards/zfs-pools.json | 173 ++++++++++++++++++++++++++-------------------- 4 files changed, 275 insertions(+), 215 deletions(-) (limited to 'dashboards') diff --git a/dashboards/gen_smart.py b/dashboards/gen_smart.py index 2cc0568..de9ab34 100644 --- a/dashboards/gen_smart.py +++ b/dashboards/gen_smart.py @@ -54,8 +54,14 @@ PASS_MAP = [{ }, }] -ATA = 'device=~"$ata_device"' -SAS = 'device=~"$sas_device"' +# Every query on this dashboard is scoped to exactly one machine. The host variable is +# deliberately single-select with no "All": `sda` is not a unique name across a fleet, +# and the `max by (device)` reductions and joinByField transformations below would +# silently merge two different physical drives into one row if more than one host were +# ever selected at a time. +HOST = 'host=~"$host"' +ATA = f'{HOST}, device=~"$ata_device"' +SAS = f'{HOST}, device=~"$sas_device"' # Splits "sde · GOOD · 6.0 TB · SAS 7.2K · HGST HUS726060AL5210 · NAHPJS45" into a # selectable value (the leading device name) and display text (the whole line). The @@ -80,7 +86,7 @@ def power_on(sel=""): source is larger is therefore never an over-estimate, and recovers a sane number on drives whose power-on counter under-counts or was reset. """ - braces = f"{{{sel}}}" if sel else "" + braces = "{" + ", ".join(x for x in (HOST, sel) if x) + "}" return (f'max by (device) (smartctl_device_power_on_seconds{braces} ' f'or (smart_selftest_last_lifetime_hours{braces} * 3600))') @@ -124,14 +130,14 @@ panels = [] panels.append(row("Fleet Overview", 0)) panels.append(stat( - "Healthy", [('count(smartctl:health_code == 0) or vector(0)', "drives")], + "Healthy", [(f'count(smartctl:health_code{{{HOST}}} == 0) or vector(0)', "drives")], gp(0, 1, 4, 4), colormode="background", thresholds={"mode": "absolute", "steps": [{"color": "green", "value": None}]}, desc="Drives with no failing attribute, no SCSI uncorrected errors and a passing " "SMART self-assessment.", )) panels.append(stat( - "Caution", [('count(smartctl:health_code == 1) or vector(0)', "drives")], + "Caution", [(f'count(smartctl:health_code{{{HOST}}} == 1) or vector(0)', "drives")], gp(4, 1, 4, 4), colormode="background", thresholds={"mode": "absolute", "steps": [ {"color": "green", "value": None}, {"color": "orange", "value": 1}]}, @@ -139,18 +145,19 @@ panels.append(stat( "defect list on SAS. Not failing yet — worth watching the trend.", )) panels.append(stat( - "Failing", [('count(smartctl:health_code == 2) or vector(0)', "drives")], + "Failing", [(f'count(smartctl:health_code{{{HOST}}} == 2) or vector(0)', "drives")], gp(8, 1, 4, 4), colormode="background", thresholds=OK_BAD, desc="SMART self-assessment failed, an attribute is at or below its threshold, or " "the drive reported errors it could not correct. Replace these.", )) panels.append(stat( - "Drives", [('count(smartctl_device_smart_status)', "total")], + "Drives", [(f'count(smartctl_device_smart_status{{{HOST}}})', "total")], gp(12, 1, 4, 4), )) panels.append(stat( "Hottest Drive", - [('max(smartctl_device_temperature{temperature_type="current"})', "max")], + [(f'max(smartctl_device_temperature{{{HOST}, temperature_type="current"}})', + "max")], gp(16, 1, 4, 4), unit="celsius", colormode="background", thresholds={"mode": "absolute", "steps": [ {"color": "green", "value": None}, @@ -167,12 +174,12 @@ panels.append(table( "All Drives", [ # group_left carries the descriptive labels from the info metric onto the # verdict, so this one query supplies device, model, serial and protocol. - ('smartctl:health_code * on(device) group_left(model_name, serial_number, ' - 'protocol) smartctl_device', None), - ('max by (device) (smartctl_device_temperature{temperature_type="current"})', - None), + (f'smartctl:health_code{{{HOST}}} * on(device) group_left(model_name, ' + f'serial_number, protocol) smartctl_device{{{HOST}}}', None), + (f'max by (device) (smartctl_device_temperature{{{HOST}, ' + f'temperature_type="current"}})', None), (power_on(), None), - ('max by (device) (smartctl_device_capacity_bytes)', None), + (f'max by (device) (smartctl_device_capacity_bytes{{{HOST}}})', None), ], gp(0, 5, 16, 10), keep="device|model_name|serial_number|protocol|Value.*", renames={ @@ -204,7 +211,8 @@ panels.append(table( panels.append(timeseries( "Temperatures", - [('smartctl_device_temperature{temperature_type="current"}', "{{device}}")], + [(f'smartctl_device_temperature{{{HOST}, temperature_type="current"}}', + "{{device}}")], gp(16, 5, 8, 10), unit="celsius", desc="Sustained operation above ~45C measurably shortens drive life.", )) @@ -418,22 +426,23 @@ panels.append(row("Self-Tests", 53)) panels.append(stat( "Drives Never Tested", - [('count(smartctl_device_smart_status) - ' - 'count(count by (device) (smart_selftest_last_lifetime_hours)) or ' - 'count(smartctl_device_smart_status)', "drives")], + [(f'count(smartctl_device_smart_status{{{HOST}}}) - ' + f'count(count by (device) (smart_selftest_last_lifetime_hours{{{HOST}}})) or ' + f'count(smartctl_device_smart_status{{{HOST}}})', "drives")], gp(0, 54, 4, 5), thresholds=OK_BAD, colormode="background", desc="Drives with no self-test log entries at all. smartd only runs tests if its " "config includes an -s schedule; merely enabling the service does not.", )) panels.append(stat( "Failed Self-Tests", - [('count(smart_selftest_last_passed == 0) or vector(0)', "tests")], + [(f'count(smart_selftest_last_passed{{{HOST}}} == 0) or vector(0)', "tests")], gp(4, 54, 4, 5), thresholds=OK_BAD, colormode="background", )) panels.append(stat( "Oldest Long Test", [(f'max({power_on()} / 3600 - on(device) group_right() ' - f'smart_selftest_last_lifetime_hours{{test_type="extended"}})', "hours ago")], + f'smart_selftest_last_lifetime_hours{{{HOST}, test_type="extended"}})', + "hours ago")], gp(8, 54, 4, 5), unit="h", thresholds={"mode": "absolute", "steps": [ {"color": "green", "value": None}, @@ -461,14 +470,14 @@ def _keyed(expr): panels.append(table( "Self-Test History", [ # Only this query keeps device and test_type as their own columns. - ('label_join(smart_selftest_last_passed, "drive_test", "/", "device", ' - '"test_type")', None), - (_keyed('smart_selftest_last_lifetime_hours'), None), - (_keyed('smart_selftest_count'), None), + (f'label_join(smart_selftest_last_passed{{{HOST}}}, "drive_test", "/", ' + f'"device", "test_type")', None), + (_keyed(f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None), + (_keyed(f'smart_selftest_count{{{HOST}}}'), None), # Powered-on hours elapsed since that test ran. group_right() keeps the # right-hand side's labels, so device and test_type survive the subtraction. (_keyed(power_on() + ' / 3600 - on(device) group_right() ' - 'smart_selftest_last_lifetime_hours'), None), + f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None), ], gp(12, 54, 12, 10), keep="drive_test|device|test_type|Value.*", renames={ @@ -501,22 +510,22 @@ panels.append(table( panels.append(row("Attribute Trends", 64, collapsed=True, children=[ timeseries( "Reallocated Sectors (all SATA drives)", - [('smartctl_device_attribute{attribute_id="5", attribute_value_type="raw"}', - "{{device}}")], + [(f'smartctl_device_attribute{{{HOST}, attribute_id="5", ' + f'attribute_value_type="raw"}}', "{{device}}")], gp(0, 65, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, ), timeseries( "Pending & Uncorrectable Sectors", - [('smartctl_device_attribute{attribute_id=~"197|198", ' - 'attribute_value_type="raw"}', "{{device}} {{attribute_name}}")], + [(f'smartctl_device_attribute{{{HOST}, attribute_id=~"197|198", ' + f'attribute_value_type="raw"}}', "{{device}} {{attribute_name}}")], gp(8, 65, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, ), timeseries( "CRC Errors (cabling)", - [('smartctl_device_attribute{attribute_id="199", attribute_value_type="raw"}', - "{{device}}")], + [(f'smartctl_device_attribute{{{HOST}, attribute_id="199", ' + f'attribute_value_type="raw"}}', "{{device}}")], gp(16, 65, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, desc="Attribute 199 counts errors on the SATA link rather than the platters. " @@ -524,14 +533,14 @@ panels.append(row("Attribute Trends", 64, collapsed=True, children=[ ), timeseries( "Data Written / Read (SAS)", [ - ('smartctl_device_bytes_written', "{{device}} written"), - ('smartctl_device_bytes_read', "{{device}} read"), + (f'smartctl_device_bytes_written{{{HOST}}}', "{{device}} written"), + (f'smartctl_device_bytes_read{{{HOST}}}', "{{device}} read"), ], gp(0, 72, 12, 7), unit="bytes", minval=0, ), timeseries( "Health Code History", - [('smartctl:health_code', "{{device}}")], + [(f'smartctl:health_code{{{HOST}}}', "{{device}}")], gp(12, 72, 12, 7), mappings=HEALTH_MAP, minval=0, maxval=2, custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"}, ), @@ -540,7 +549,7 @@ panels.append(row("Attribute Trends", 64, collapsed=True, children=[ panels.append(row("Collector Health", 65, collapsed=True, children=[ timeseries( "smartctl Exit Status", - [('smartctl_device_smartctl_exit_status', "{{device}}")], + [(f'smartctl_device_smartctl_exit_status{{{HOST}}}', "{{device}}")], gp(0, 66, 8, 7), minval=0, desc="Non-zero means smartctl itself had trouble reading the device. Bit 0 is " "a command-line problem, bit 1 a device-open failure, bit 3 a failing " @@ -548,13 +557,13 @@ panels.append(row("Collector Health", 65, collapsed=True, children=[ ), timeseries( "Self-Test Collector Duration", - [('smart_selftest_collection_duration_seconds', "collection")], + [(f'smart_selftest_collection_duration_seconds{{{HOST}}}', "collection")], gp(8, 66, 8, 7), unit="s", minval=0, ), timeseries( "Devices Seen", [ - ('smartctl_devices', "exporter"), - ('smart_selftest_devices', "self-test collector"), + (f'smartctl_devices{{{HOST}}}', "exporter"), + (f'smart_selftest_devices{{{HOST}}}', "self-test collector"), ], gp(16, 66, 8, 7), minval=0, desc="These should agree. A divergence means one of the two cannot see some " @@ -575,9 +584,14 @@ print(json.dumps(dashboard( # `device=~"$ata_device"` selector below is unaffected, while the *text* the UI # shows -- and that ${ata_device:text} puts in the row title -- is the whole # line. That is what makes a collapsed row identifiable without opening it. - query_var("ata_device", 'label_values(smartctl:row{bus="ata"}, row)', + # One machine at a time. Not multi-select and no "All" -- see the comment + # on HOST above for why selecting two hosts at once would silently merge + # same-named drives rather than showing both. + query_var("host", "label_values(smartctl_device_smart_status, host)", + label="Host", multi=False, include_all=False), + query_var("ata_device", 'label_values(smartctl:row{bus="ata", host=~"$host"}, row)', label="SATA drive", regex=ROW_REGEX), - query_var("sas_device", 'label_values(smartctl:row{bus="sas"}, row)', + query_var("sas_device", 'label_values(smartctl:row{bus="sas", host=~"$host"}, row)', label="SAS drive", regex=ROW_REGEX), ], tags=["smart", "storage", "hardware"], diff --git a/dashboards/gen_zfs.py b/dashboards/gen_zfs.py index 4edb38b..0057506 100644 --- a/dashboards/gen_zfs.py +++ b/dashboards/gen_zfs.py @@ -20,7 +20,8 @@ If you add a panel, add the metric to the collector first and keep this list hon import json -from panels import (DS, OK_BAD, bargauge, gauge, gp, row, stat, table, timeseries) +from panels import (DS, OK_BAD, bargauge, gauge, gp, query_var, row, stat, table, + timeseries) STATE_MAP = [{ @@ -46,7 +47,12 @@ BOOL_MAP = [{ }, }] -POOL = 'pool=~"$pool"' +# Every query on this dashboard is scoped to exactly one machine. The host variable is +# single-select with no "All" on purpose: pool names are not unique across a fleet, and +# the `max by (pool)` reductions and joinByField transformations below would silently +# merge two different pools that happen to share a name. +HOST = 'host=~"$host"' +POOL = f'{HOST}, pool=~"$pool"' panels = [] # ====================================================================================== @@ -124,7 +130,7 @@ panels.append(stat( panels.append(stat( "Snapshots Overdue", [(f'count((time() - zfs_snapshot_newest_timestamp_seconds{{{POOL}}}) > on(label) ' - f'group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)', "overdue")], + f'group_left() (2 * zfs_snapshot_interval_seconds{{{HOST}}})) or vector(0)', "overdue")], gp(15, 5, 4, 4), thresholds=OK_BAD, colormode="background", desc="Dataset/label pairs whose newest snapshot is more than two intervals old — " "i.e. the auto-snapshot timer has missed at least one run.", @@ -294,7 +300,7 @@ panels.append(bargauge( panels.append(bargauge( "Snapshots vs Retention Target", [(f'min by (label) (zfs_snapshot_count{{{POOL}}} / on(label) group_left() ' - f'zfs_snapshot_retention_target)', "{{label}}")], + f'zfs_snapshot_retention_target{{{HOST}}})', "{{label}}")], gp(8, 43, 8, 8), unit="percentunit", maxv=1.2, thresholds={"mode": "absolute", "steps": [ {"color": "red", "value": None}, @@ -329,15 +335,16 @@ panels.append(timeseries( panels.append(stat( "Snapshot / Scrub Job Failures", - [('sum(node_systemd_unit_state{name=~"zfs-(snapshot|scrub).*",state="failed"}) ' - 'or vector(0)', "failed units")], + [(f'sum(node_systemd_unit_state{{{HOST}, name=~"zfs-(snapshot|scrub).*", ' + f'state="failed"}}) or vector(0)', "failed units")], gp(0, 59, 6, 6), thresholds=OK_BAD, colormode="background", desc="systemd units for the auto-snapshot and auto-scrub timers currently in a " "failed state.", )) panels.append(bargauge( "Time Since Timer Last Fired", - [('time() - node_systemd_timer_last_trigger_seconds{name=~"zfs-.*"}', "{{name}}")], + [(f'time() - node_systemd_timer_last_trigger_seconds{{{HOST}, ' + f'name=~"zfs-.*"}}', "{{name}}")], gp(6, 59, 18, 6), unit="s", desc="When each zfs-* systemd timer last triggered. This is the job side of the " "picture; the snapshot ages above are the result side.", @@ -445,9 +452,9 @@ panels.append(row("Performance", 91)) panels.append(timeseries( "Dataset Throughput", [ - (f'rate(node_zfs_zpool_dataset_nread{{zpool=~"$pool"}}[$__rate_interval])', + (f'rate(node_zfs_zpool_dataset_nread{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} read"), - (f'-rate(node_zfs_zpool_dataset_nwritten{{zpool=~"$pool"}}[$__rate_interval])', + (f'-rate(node_zfs_zpool_dataset_nwritten{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} write"), ], gp(0, 92, 12, 8), unit="Bps", @@ -456,9 +463,9 @@ panels.append(timeseries( )) panels.append(timeseries( "Dataset IOPS", [ - (f'rate(node_zfs_zpool_dataset_reads{{zpool=~"$pool"}}[$__rate_interval])', + (f'rate(node_zfs_zpool_dataset_reads{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} read"), - (f'-rate(node_zfs_zpool_dataset_writes{{zpool=~"$pool"}}[$__rate_interval])', + (f'-rate(node_zfs_zpool_dataset_writes{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} write"), ], gp(12, 92, 12, 8), unit="iops", @@ -467,24 +474,24 @@ panels.append(timeseries( panels.append(timeseries( "ARC Size", [ - ("node_zfs_arc_size", "size"), - ("node_zfs_arc_c_max", "target max (c_max)"), - ("node_zfs_arc_c_min", "target min (c_min)"), + (f"node_zfs_arc_size{{{HOST}}}", "size"), + (f"node_zfs_arc_c_max{{{HOST}}}", "target max (c_max)"), + (f"node_zfs_arc_c_min{{{HOST}}}", "target min (c_min)"), ], gp(0, 100, 8, 8), unit="bytes", minval=0, )) panels.append(timeseries( "ARC Hit Ratio", - [("rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min(" - "rate(node_zfs_arc_hits[$__rate_interval]) + " - "rate(node_zfs_arc_misses[$__rate_interval]), 1)", "hit ratio")], + [(f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval]) / clamp_min(" + f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval]) + " + f"rate(node_zfs_arc_misses{{{HOST}}}[$__rate_interval]), 1)", "hit ratio")], gp(8, 100, 8, 8), unit="percentunit", minval=0, desc="clamp_min avoids a divide-by-zero while the pool is idle.", )) panels.append(timeseries( "ARC Hits / Misses", [ - ("rate(node_zfs_arc_hits[$__rate_interval])", "hits"), - ("rate(node_zfs_arc_misses[$__rate_interval])", "misses"), + (f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval])", "hits"), + (f"rate(node_zfs_arc_misses{{{HOST}}}[$__rate_interval])", "misses"), ], gp(16, 100, 8, 8), unit="ops", minval=0, )) @@ -553,7 +560,7 @@ panels.append(row("Pool Internals", 108, collapsed=True, children=[ panels.append(row("Collector Health", 109, collapsed=True, children=[ timeseries( "ZFS Collector Duration", - [("zfs_metrics_collection_duration_seconds", "collection")], + [(f"zfs_metrics_collection_duration_seconds{{{HOST}}}", "collection")], gp(0, 110, 8, 7), unit="s", minval=0, desc="How long collectors/zfs-metrics.py takes. Listing every snapshot is the " "slow part; if this approaches the 1 minute timer interval, raise " @@ -561,7 +568,7 @@ panels.append(row("Collector Health", 109, collapsed=True, children=[ ), timeseries( "Textfile Age", - [("time() - node_textfile_mtime_seconds", "{{file}}")], + [(f"time() - node_textfile_mtime_seconds{{{HOST}}}", "{{file}}")], gp(8, 110, 8, 7), unit="s", minval=0, desc="Age of each .prom file. If this climbs without bound the collector has " "stopped writing and every ZFS metric above is stale.", @@ -569,7 +576,7 @@ panels.append(row("Collector Health", 109, collapsed=True, children=[ timeseries( "Scrape Health", [ ('up{job="node"}', "node exporter up"), - ("node_textfile_scrape_error", "textfile parse error"), + (f"node_textfile_scrape_error{{{HOST}}}", "textfile parse error"), ], gp(16, 110, 8, 7), minval=0, desc="node_textfile_scrape_error goes to 1 if a .prom file is malformed, in " @@ -591,21 +598,16 @@ dashboard = { "graphTooltip": 1, "refresh": "1m", "time": {"from": "now-24h", "to": "now"}, - "templating": {"list": [{ - "name": "pool", - "label": "Pool", - "type": "query", - "datasource": DS, - "query": {"qryType": 1, "query": "label_values(zfs_pool_health_code, pool)", - "refId": "PrometheusVariableQueryEditor-VariableQuery"}, - "refresh": 1, - "includeAll": True, - "allValue": ".*", - "multi": True, - "current": {"text": ["All"], "value": ["$__all"]}, - "options": [], - "sort": 1, - }]}, + "templating": {"list": [ + # One machine at a time: not multi-select and no "All". See the comment on HOST + # above for why two hosts at once would silently merge same-named pools. + query_var("host", "label_values(zfs_pool_health_code, host)", + label="Host", multi=False, include_all=False), + # Pools are listed for the selected host only, so switching hosts re-populates + # the picker rather than offering pools that do not exist on this machine. + query_var("pool", 'label_values(zfs_pool_health_code{host=~"$host"}, pool)', + label="Pool"), + ]}, "annotations": {"list": [{ "builtIn": 1, "datasource": {"type": "datasource", "uid": "grafana"}, "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/dashboards/smart.json b/dashboards/smart.json index ed48cd1..f6c9867 100644 --- a/dashboards/smart.json +++ b/dashboards/smart.json @@ -19,6 +19,27 @@ }, "templating": { "list": [ + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(smartctl_device_smart_status, host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [], + "sort": 1, + "hide": 0, + "regex": "" + }, { "name": "ata_device", "label": "SATA drive", @@ -29,7 +50,7 @@ }, "query": { "qryType": 1, - "query": "label_values(smartctl:row{bus=\"ata\"}, row)", + "query": "label_values(smartctl:row{bus=\"ata\", host=~\"$host\"}, row)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -59,7 +80,7 @@ }, "query": { "qryType": 1, - "query": "label_values(smartctl:row{bus=\"sas\"}, row)", + "query": "label_values(smartctl:row{bus=\"sas\", host=~\"$host\"}, row)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -166,7 +187,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smartctl:health_code == 0) or vector(0)", + "expr": "count(smartctl:health_code{host=~\"$host\"} == 0) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -233,7 +254,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smartctl:health_code == 1) or vector(0)", + "expr": "count(smartctl:health_code{host=~\"$host\"} == 1) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -300,7 +321,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smartctl:health_code == 2) or vector(0)", + "expr": "count(smartctl:health_code{host=~\"$host\"} == 2) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -363,7 +384,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smartctl_device_smart_status)", + "expr": "count(smartctl_device_smart_status{host=~\"$host\"})", "refId": "A", "instant": true, "range": false, @@ -435,7 +456,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max(smartctl_device_temperature{temperature_type=\"current\"})", + "expr": "max(smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"})", "refId": "A", "instant": true, "range": false, @@ -499,7 +520,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)))", + "expr": "max(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)))", "refId": "A", "instant": true, "range": false, @@ -664,7 +685,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl:health_code * on(device) group_left(model_name, serial_number, protocol) smartctl_device", + "expr": "smartctl:health_code{host=~\"$host\"} * on(device) group_left(model_name, serial_number, protocol) smartctl_device{host=~\"$host\"}", "refId": "A", "instant": true, "range": false, @@ -676,7 +697,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_temperature{temperature_type=\"current\"})", + "expr": "max by (device) (smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"})", "refId": "B", "instant": true, "range": false, @@ -688,7 +709,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600))", + "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600))", "refId": "C", "instant": true, "range": false, @@ -700,7 +721,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_capacity_bytes)", + "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\"})", "refId": "D", "instant": true, "range": false, @@ -841,7 +862,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_temperature{temperature_type=\"current\"}", + "expr": "smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"}", "refId": "A", "instant": false, "range": true, @@ -948,7 +969,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl:health_code{device=~\"$ata_device\"}", + "expr": "smartctl:health_code{host=~\"$host\", device=~\"$ata_device\"}", "refId": "A", "instant": true, "range": false, @@ -1020,7 +1041,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_temperature{device=~\"$ata_device\", temperature_type=\"current\"}", + "expr": "smartctl_device_temperature{host=~\"$host\", device=~\"$ata_device\", temperature_type=\"current\"}", "refId": "A", "instant": true, "range": false, @@ -1084,7 +1105,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$ata_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$ata_device\"} * 3600))", + "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\", host=~\"$host\", device=~\"$ata_device\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\", host=~\"$host\", device=~\"$ata_device\"} * 3600))", "refId": "A", "instant": true, "range": false, @@ -1148,7 +1169,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_capacity_bytes{device=~\"$ata_device\"}", + "expr": "smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$ata_device\"}", "refId": "A", "instant": true, "range": false, @@ -1211,7 +1232,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_power_cycle_count{device=~\"$ata_device\"}", + "expr": "smartctl_device_power_cycle_count{host=~\"$host\", device=~\"$ata_device\"}", "refId": "A", "instant": true, "range": false, @@ -1278,7 +1299,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"5\", attribute_value_type=\"raw\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_id=\"5\", attribute_value_type=\"raw\"}", "refId": "A", "instant": true, "range": false, @@ -1345,7 +1366,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"197\", attribute_value_type=\"raw\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_id=\"197\", attribute_value_type=\"raw\"}", "refId": "A", "instant": true, "range": false, @@ -1442,7 +1463,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device{device=~\"$ata_device\"}", + "expr": "smartctl_device{host=~\"$host\", device=~\"$ata_device\"}", "refId": "A", "instant": true, "range": false, @@ -1454,7 +1475,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$ata_device\"})", + "expr": "max by (device) (smartctl_device_rotation_rate{host=~\"$host\", device=~\"$ata_device\"})", "refId": "B", "instant": true, "range": false, @@ -1466,7 +1487,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$ata_device\"})", + "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$ata_device\"})", "refId": "C", "instant": true, "range": false, @@ -1623,7 +1644,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"value\"}", "refId": "A", "instant": true, "range": false, @@ -1635,7 +1656,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"worst\"})", + "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"worst\"})", "refId": "B", "instant": true, "range": false, @@ -1647,7 +1668,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})", + "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"thresh\"})", "refId": "C", "instant": true, "range": false, @@ -1659,7 +1680,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"raw\"})", + "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"raw\"})", "refId": "D", "instant": true, "range": false, @@ -1671,7 +1692,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"} - on(device, attribute_id) smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})", + "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"value\"} - on(device, attribute_id) smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"thresh\"})", "refId": "E", "instant": true, "range": false, @@ -1831,7 +1852,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl:health_code{device=~\"$sas_device\"}", + "expr": "smartctl:health_code{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": true, "range": false, @@ -1903,7 +1924,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_temperature{device=~\"$sas_device\", temperature_type=\"current\"}", + "expr": "smartctl_device_temperature{host=~\"$host\", device=~\"$sas_device\", temperature_type=\"current\"}", "refId": "A", "instant": true, "range": false, @@ -1967,7 +1988,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$sas_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$sas_device\"} * 3600))", + "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\", host=~\"$host\", device=~\"$sas_device\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\", host=~\"$host\", device=~\"$sas_device\"} * 3600))", "refId": "A", "instant": true, "range": false, @@ -2031,7 +2052,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_capacity_bytes{device=~\"$sas_device\"}", + "expr": "smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": true, "range": false, @@ -2095,7 +2116,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_interface_speed{device=~\"$sas_device\", speed_type=\"current\"} * 1000", + "expr": "smartctl_device_interface_speed{host=~\"$host\", device=~\"$sas_device\", speed_type=\"current\"} * 1000", "refId": "A", "instant": true, "range": false, @@ -2162,7 +2183,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}", + "expr": "smartctl_scsi_grown_defect_list{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": true, "range": false, @@ -2229,7 +2250,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"} + smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}", + "expr": "smartctl_read_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"} + smartctl_write_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": true, "range": false, @@ -2326,7 +2347,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device{device=~\"$sas_device\"}", + "expr": "smartctl_device{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": true, "range": false, @@ -2338,7 +2359,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$sas_device\"})", + "expr": "max by (device) (smartctl_device_rotation_rate{host=~\"$host\", device=~\"$sas_device\"})", "refId": "B", "instant": true, "range": false, @@ -2350,7 +2371,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$sas_device\"})", + "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$sas_device\"})", "refId": "C", "instant": true, "range": false, @@ -2485,7 +2506,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_read_errors_corrected_by_eccfast{device=~\"$sas_device\"}", + "expr": "smartctl_read_errors_corrected_by_eccfast{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": false, "range": true, @@ -2497,7 +2518,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_read_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}", + "expr": "smartctl_read_errors_corrected_by_eccdelayed{host=~\"$host\", device=~\"$sas_device\"}", "refId": "B", "instant": false, "range": true, @@ -2509,7 +2530,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_read_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}", + "expr": "smartctl_read_errors_corrected_by_rereads_rewrites{host=~\"$host\", device=~\"$sas_device\"}", "refId": "C", "instant": false, "range": true, @@ -2521,7 +2542,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_write_errors_corrected_by_eccfast{device=~\"$sas_device\"}", + "expr": "smartctl_write_errors_corrected_by_eccfast{host=~\"$host\", device=~\"$sas_device\"}", "refId": "D", "instant": false, "range": true, @@ -2533,7 +2554,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_write_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}", + "expr": "smartctl_write_errors_corrected_by_eccdelayed{host=~\"$host\", device=~\"$sas_device\"}", "refId": "E", "instant": false, "range": true, @@ -2545,7 +2566,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_write_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}", + "expr": "smartctl_write_errors_corrected_by_rereads_rewrites{host=~\"$host\", device=~\"$sas_device\"}", "refId": "F", "instant": false, "range": true, @@ -2631,7 +2652,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}", + "expr": "smartctl_scsi_grown_defect_list{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": false, "range": true, @@ -2643,7 +2664,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"}", + "expr": "smartctl_read_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}", "refId": "B", "instant": false, "range": true, @@ -2655,7 +2676,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}", + "expr": "smartctl_write_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}", "refId": "C", "instant": false, "range": true, @@ -2735,7 +2756,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smartctl_device_smart_status) - count(count by (device) (smart_selftest_last_lifetime_hours)) or count(smartctl_device_smart_status)", + "expr": "count(smartctl_device_smart_status{host=~\"$host\"}) - count(count by (device) (smart_selftest_last_lifetime_hours{host=~\"$host\"})) or count(smartctl_device_smart_status{host=~\"$host\"})", "refId": "A", "instant": true, "range": false, @@ -2802,7 +2823,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smart_selftest_last_passed == 0) or vector(0)", + "expr": "count(smart_selftest_last_passed{host=~\"$host\"} == 0) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -2875,7 +2896,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{test_type=\"extended\"})", + "expr": "max(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{host=~\"$host\", test_type=\"extended\"})", "refId": "A", "instant": true, "range": false, @@ -2979,7 +3000,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "label_join(smart_selftest_last_passed, \"drive_test\", \"/\", \"device\", \"test_type\")", + "expr": "label_join(smart_selftest_last_passed{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\")", "refId": "A", "instant": true, "range": false, @@ -2991,7 +3012,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (drive_test) (label_join(smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))", + "expr": "max by (drive_test) (label_join(smart_selftest_last_lifetime_hours{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))", "refId": "B", "instant": true, "range": false, @@ -3003,7 +3024,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (drive_test) (label_join(smart_selftest_count, \"drive_test\", \"/\", \"device\", \"test_type\"))", + "expr": "max by (drive_test) (label_join(smart_selftest_count{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))", "refId": "C", "instant": true, "range": false, @@ -3015,7 +3036,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (drive_test) (label_join(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))", + "expr": "max by (drive_test) (label_join(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))", "refId": "D", "instant": true, "range": false, @@ -3166,7 +3187,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{attribute_id=\"5\", attribute_value_type=\"raw\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=\"5\", attribute_value_type=\"raw\"}", "refId": "A", "instant": false, "range": true, @@ -3252,7 +3273,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{attribute_id=~\"197|198\", attribute_value_type=\"raw\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=~\"197|198\", attribute_value_type=\"raw\"}", "refId": "A", "instant": false, "range": true, @@ -3338,7 +3359,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{attribute_id=\"199\", attribute_value_type=\"raw\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=\"199\", attribute_value_type=\"raw\"}", "refId": "A", "instant": false, "range": true, @@ -3425,7 +3446,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_bytes_written", + "expr": "smartctl_device_bytes_written{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3437,7 +3458,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_bytes_read", + "expr": "smartctl_device_bytes_read{host=~\"$host\"}", "refId": "B", "instant": false, "range": true, @@ -3545,7 +3566,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl:health_code", + "expr": "smartctl:health_code{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3645,7 +3666,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_smartctl_exit_status", + "expr": "smartctl_device_smartctl_exit_status{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3732,7 +3753,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smart_selftest_collection_duration_seconds", + "expr": "smart_selftest_collection_duration_seconds{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3818,7 +3839,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_devices", + "expr": "smartctl_devices{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3830,7 +3851,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smart_selftest_devices", + "expr": "smart_selftest_devices{host=~\"$host\"}", "refId": "B", "instant": false, "range": true, diff --git a/dashboards/zfs-pools.json b/dashboards/zfs-pools.json index 0721bb8..c36a42a 100644 --- a/dashboards/zfs-pools.json +++ b/dashboards/zfs-pools.json @@ -18,6 +18,27 @@ }, "templating": { "list": [ + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(zfs_pool_health_code, host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [], + "sort": 1, + "hide": 0, + "regex": "" + }, { "name": "pool", "label": "Pool", @@ -28,13 +49,17 @@ }, "query": { "qryType": 1, - "query": "label_values(zfs_pool_health_code, pool)", + "query": "label_values(zfs_pool_health_code{host=~\"$host\"}, pool)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "includeAll": true, - "allValue": ".*", "multi": true, + "options": [], + "sort": 1, + "hide": 0, + "regex": "", + "allValue": ".*", "current": { "text": [ "All" @@ -42,9 +67,7 @@ "value": [ "$__all" ] - }, - "options": [], - "sort": 1 + } } ] }, @@ -184,7 +207,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_health_code{pool=~\"$pool\"}", + "expr": "zfs_pool_health_code{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -251,7 +274,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(zfs_pool_data_errors{pool=~\"$pool\"}) or vector(0)", + "expr": "sum(zfs_pool_data_errors{host=~\"$host\", pool=~\"$pool\"}) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -318,7 +341,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(zfs_pool_vdev_errors_total{pool=~\"$pool\"}) or vector(0)", + "expr": "sum(zfs_pool_vdev_errors_total{host=~\"$host\", pool=~\"$pool\"}) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -390,7 +413,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - max(zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"})", + "expr": "time() - max(zfs_pool_last_scrub_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, @@ -469,7 +492,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max(zfs_pool_scrub_in_progress{pool=~\"$pool\"})", + "expr": "max(zfs_pool_scrub_in_progress{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, @@ -481,7 +504,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max(zfs_pool_resilver_in_progress{pool=~\"$pool\"})", + "expr": "max(zfs_pool_resilver_in_progress{host=~\"$host\", pool=~\"$pool\"})", "refId": "B", "instant": true, "range": false, @@ -548,7 +571,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(zfs_dataset_encrypted{pool=~\"$pool\"} == 1 and zfs_dataset_key_available{pool=~\"$pool\"} == 0) or vector(0)", + "expr": "count(zfs_dataset_encrypted{host=~\"$host\", pool=~\"$pool\"} == 1 and zfs_dataset_key_available{host=~\"$host\", pool=~\"$pool\"} == 0) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -618,7 +641,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_capacity_ratio{pool=~\"$pool\"}", + "expr": "zfs_pool_capacity_ratio{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -688,7 +711,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}", + "expr": "zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -752,7 +775,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -819,7 +842,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count((time() - zfs_snapshot_newest_timestamp_seconds{pool=~\"$pool\"}) > on(label) group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)", + "expr": "count((time() - zfs_snapshot_newest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"}) > on(label) group_left() (2 * zfs_snapshot_interval_seconds{host=~\"$host\"})) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -883,7 +906,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})", + "expr": "time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, @@ -970,7 +993,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_allocated_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_allocated_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -982,7 +1005,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "B", "instant": false, "range": true, @@ -1070,7 +1093,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}", + "expr": "zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -1206,7 +1229,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_size_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -1218,7 +1241,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_allocated_bytes{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_allocated_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "B", "instant": true, "range": false, @@ -1230,7 +1253,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_free_bytes{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "C", "instant": true, "range": false, @@ -1242,7 +1265,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_capacity_ratio{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_capacity_ratio{host=~\"$host\", pool=~\"$pool\"})", "refId": "D", "instant": true, "range": false, @@ -1254,7 +1277,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_fragmentation_ratio{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"})", "refId": "E", "instant": true, "range": false, @@ -1266,7 +1289,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_snapshot_used_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "F", "instant": true, "range": false, @@ -1278,7 +1301,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_dedup_ratio{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_dedup_ratio{host=~\"$host\", pool=~\"$pool\"})", "refId": "G", "instant": true, "range": false, @@ -1481,7 +1504,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_vdev_state_code{pool=~\"$pool\"}", + "expr": "zfs_pool_vdev_state_code{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -1589,7 +1612,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_vdev_errors_total{pool=~\"$pool\"}", + "expr": "zfs_pool_vdev_errors_total{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -1737,7 +1760,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"} * 1000", + "expr": "zfs_pool_last_scrub_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"} * 1000", "refId": "A", "instant": true, "range": false, @@ -1749,7 +1772,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_last_scrub_duration_seconds{pool=~\"$pool\"}", + "expr": "zfs_pool_last_scrub_duration_seconds{host=~\"$host\", pool=~\"$pool\"}", "refId": "B", "instant": true, "range": false, @@ -1761,7 +1784,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_last_scrub_errors{pool=~\"$pool\"}", + "expr": "zfs_pool_last_scrub_errors{host=~\"$host\", pool=~\"$pool\"}", "refId": "C", "instant": true, "range": false, @@ -1773,7 +1796,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_last_scrub_repaired_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_last_scrub_repaired_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "D", "instant": true, "range": false, @@ -1785,7 +1808,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_last_resilver_end_timestamp_seconds{pool=~\"$pool\"} * 1000", + "expr": "zfs_pool_last_resilver_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"} * 1000", "refId": "E", "instant": true, "range": false, @@ -1903,7 +1926,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_data_errors{pool=~\"$pool\"}", + "expr": "zfs_pool_data_errors{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -1987,7 +2010,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{pool=~\"$pool\"})", + "expr": "time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, @@ -2059,7 +2082,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "min by (label) (zfs_snapshot_count{pool=~\"$pool\"} / on(label) group_left() zfs_snapshot_retention_target)", + "expr": "min by (label) (zfs_snapshot_count{host=~\"$host\", pool=~\"$pool\"} / on(label) group_left() zfs_snapshot_retention_target{host=~\"$host\"})", "refId": "A", "instant": true, "range": false, @@ -2122,7 +2145,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - min by (label) (zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})", + "expr": "time() - min by (label) (zfs_pool_snapshot_oldest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, @@ -2208,7 +2231,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (label) (zfs_snapshot_count{pool=~\"$pool\"})", + "expr": "sum by (label) (zfs_snapshot_count{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": false, "range": true, @@ -2295,7 +2318,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (label) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})", + "expr": "sum by (label) (zfs_pool_snapshot_used_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": false, "range": true, @@ -2362,7 +2385,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(node_systemd_unit_state{name=~\"zfs-(snapshot|scrub).*\",state=\"failed\"}) or vector(0)", + "expr": "sum(node_systemd_unit_state{host=~\"$host\", name=~\"zfs-(snapshot|scrub).*\", state=\"failed\"}) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -2425,7 +2448,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - node_systemd_timer_last_trigger_seconds{name=~\"zfs-.*\"}", + "expr": "time() - node_systemd_timer_last_trigger_seconds{host=~\"$host\", name=~\"zfs-.*\"}", "refId": "A", "instant": true, "range": false, @@ -2525,7 +2548,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}", + "expr": "zfs_dataset_used_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -2630,7 +2653,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}", + "expr": "zfs_dataset_used_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -2642,7 +2665,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_used_by_dataset_bytes{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_used_by_dataset_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "B", "instant": true, "range": false, @@ -2654,7 +2677,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_used_by_snapshots_bytes{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_used_by_snapshots_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "C", "instant": true, "range": false, @@ -2666,7 +2689,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_referenced_bytes{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_referenced_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "D", "instant": true, "range": false, @@ -2678,7 +2701,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_logical_used_bytes{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_logical_used_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "E", "instant": true, "range": false, @@ -2690,7 +2713,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_available_bytes{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_available_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "F", "instant": true, "range": false, @@ -2702,7 +2725,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_compression_ratio{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_compression_ratio{host=~\"$host\", pool=~\"$pool\"})", "refId": "G", "instant": true, "range": false, @@ -2714,7 +2737,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_quota_bytes{pool=~\"$pool\"} > 0)", + "expr": "max by (dataset) (zfs_dataset_quota_bytes{host=~\"$host\", pool=~\"$pool\"} > 0)", "refId": "H", "instant": true, "range": false, @@ -2841,7 +2864,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_compression_ratio{pool=~\"$pool\"} * on(pool, dataset) group_left(compression) zfs_dataset_compression{pool=~\"$pool\"}", + "expr": "zfs_dataset_compression_ratio{host=~\"$host\", pool=~\"$pool\"} * on(pool, dataset) group_left(compression) zfs_dataset_compression{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -2991,7 +3014,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_keystatus_code{pool=~\"$pool\"} * on(pool, dataset) group_left(encryption) zfs_dataset_encryption{pool=~\"$pool\"}", + "expr": "zfs_dataset_keystatus_code{host=~\"$host\", pool=~\"$pool\"} * on(pool, dataset) group_left(encryption) zfs_dataset_encryption{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -3118,7 +3141,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(node_zfs_zpool_dataset_nread{zpool=~\"$pool\"}[$__rate_interval])", + "expr": "rate(node_zfs_zpool_dataset_nread{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", "refId": "A", "instant": false, "range": true, @@ -3130,7 +3153,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "-rate(node_zfs_zpool_dataset_nwritten{zpool=~\"$pool\"}[$__rate_interval])", + "expr": "-rate(node_zfs_zpool_dataset_nwritten{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", "refId": "B", "instant": false, "range": true, @@ -3216,7 +3239,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(node_zfs_zpool_dataset_reads{zpool=~\"$pool\"}[$__rate_interval])", + "expr": "rate(node_zfs_zpool_dataset_reads{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", "refId": "A", "instant": false, "range": true, @@ -3228,7 +3251,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "-rate(node_zfs_zpool_dataset_writes{zpool=~\"$pool\"}[$__rate_interval])", + "expr": "-rate(node_zfs_zpool_dataset_writes{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", "refId": "B", "instant": false, "range": true, @@ -3315,7 +3338,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "node_zfs_arc_size", + "expr": "node_zfs_arc_size{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3327,7 +3350,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "node_zfs_arc_c_max", + "expr": "node_zfs_arc_c_max{host=~\"$host\"}", "refId": "B", "instant": false, "range": true, @@ -3339,7 +3362,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "node_zfs_arc_c_min", + "expr": "node_zfs_arc_c_min{host=~\"$host\"}", "refId": "C", "instant": false, "range": true, @@ -3426,7 +3449,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min(rate(node_zfs_arc_hits[$__rate_interval]) + rate(node_zfs_arc_misses[$__rate_interval]), 1)", + "expr": "rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval]) / clamp_min(rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval]) + rate(node_zfs_arc_misses{host=~\"$host\"}[$__rate_interval]), 1)", "refId": "A", "instant": false, "range": true, @@ -3513,7 +3536,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(node_zfs_arc_hits[$__rate_interval])", + "expr": "rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval])", "refId": "A", "instant": false, "range": true, @@ -3525,7 +3548,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(node_zfs_arc_misses[$__rate_interval])", + "expr": "rate(node_zfs_arc_misses{host=~\"$host\"}[$__rate_interval])", "refId": "B", "instant": false, "range": true, @@ -3624,7 +3647,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_freeing_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_freeing_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -3636,7 +3659,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_leaked_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_leaked_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "B", "instant": false, "range": true, @@ -3648,7 +3671,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_checkpoint_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_checkpoint_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "C", "instant": false, "range": true, @@ -3734,7 +3757,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_dedup_ratio{pool=~\"$pool\"}", + "expr": "zfs_pool_dedup_ratio{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -3872,7 +3895,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_health_code{pool=~\"$pool\"}", + "expr": "zfs_pool_health_code{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -4005,7 +4028,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_recordsize_bytes{pool=~\"$pool\"}", + "expr": "zfs_dataset_recordsize_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -4017,7 +4040,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_quota_bytes{pool=~\"$pool\"}", + "expr": "zfs_dataset_quota_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "B", "instant": true, "range": false, @@ -4029,7 +4052,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_refquota_bytes{pool=~\"$pool\"}", + "expr": "zfs_dataset_refquota_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "C", "instant": true, "range": false, @@ -4041,7 +4064,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_autosnapshot_enabled{pool=~\"$pool\"}", + "expr": "zfs_dataset_autosnapshot_enabled{host=~\"$host\", pool=~\"$pool\"}", "refId": "D", "instant": true, "range": false, @@ -4173,7 +4196,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_metrics_collection_duration_seconds", + "expr": "zfs_metrics_collection_duration_seconds{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -4260,7 +4283,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - node_textfile_mtime_seconds", + "expr": "time() - node_textfile_mtime_seconds{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -4358,7 +4381,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "node_textfile_scrape_error", + "expr": "node_textfile_scrape_error{host=~\"$host\"}", "refId": "B", "instant": false, "range": true, -- cgit v1.3.1