aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTim Keller <tjk@tjkeller.xyz>2026-08-27 10:13:13 -0500
committerTim Keller <tjk@tjkeller.xyz>2026-08-27 10:13:13 -0500
commita7f38335e5fed5a98bfaad64ab40544124337224 (patch)
treed70221c4093fbe15a428d407893e740ef05711fc
downloadtelemetry-flake-a7f38335e5fed5a98bfaad64ab40544124337224.tar.xz
telemetry-flake-a7f38335e5fed5a98bfaad64ab40544124337224.zip
initial commit
-rw-r--r--collectors/smart-selftest.py330
-rw-r--r--collectors/zfs-metrics.py610
-rw-r--r--dashboards/gen_smart.py584
-rw-r--r--dashboards/gen_zfs.py617
-rw-r--r--dashboards/panels.py291
-rw-r--r--dashboards/smart.json3844
-rw-r--r--dashboards/zfs-pools.json4372
-rw-r--r--lgtm.nix534
-rw-r--r--rules/smartctl.yml116
9 files changed, 11298 insertions, 0 deletions
diff --git a/collectors/smart-selftest.py b/collectors/smart-selftest.py
new file mode 100644
index 0000000..fa9d68f
--- /dev/null
+++ b/collectors/smart-selftest.py
@@ -0,0 +1,330 @@
+#!/usr/bin/env python3
+"""Collect SMART self-test history and drive identity for node_exporter's textfile
+collector.
+
+Two things smartctl_exporter cannot give us:
+
+* Self-test history. The exporter reads only `count` and `error_count_total` out of
+ smartctl's self-test log and discards the log table itself, so it cannot tell you
+ when a drive last ran a short or an extended test, or whether that test passed.
+
+* A ready-to-display drive identity string. Grafana can only put a *label* in a row
+ title, and PromQL has no way to turn a number (capacity, rotation rate) into one.
+ So the formatting happens here, in `smart_drive_label`.
+
+Device names are emitted without the /dev/ prefix so they join against
+smartctl_exporter's `device` label, which is the bare name ("sda").
+
+Run with --output PATH. Reading the self-test log does not spin an idle disk up, but
+there is no reason to poll it often -- a 15 minute timer is plenty.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+from collections import defaultdict
+
+# smartctl spells these differently per transport and firmware; normalise to the names
+# people actually use.
+TEST_TYPES = [
+ (re.compile(r"\bshort\b", re.I), "short"),
+ (re.compile(r"\b(extended|long)\b", re.I), "extended"),
+ (re.compile(r"\bconveyance\b", re.I), "conveyance"),
+ (re.compile(r"\bselective\b", re.I), "selective"),
+ (re.compile(r"\boffline\b", re.I), "offline"),
+]
+
+
+def warn(msg: str) -> None:
+ print(f"smart-selftest: {msg}", file=sys.stderr)
+
+
+def render_labels(labels: dict) -> str:
+ if not labels:
+ return ""
+ parts = [f'{k}="{_clean(str(v))}"' for k, v in sorted(labels.items())]
+ return "{" + ",".join(parts) + "}"
+
+
+def _clean(value: str) -> str:
+ return value.translate(str.maketrans("", "", '"\\\n'))
+
+
+class Registry:
+ def __init__(self) -> None:
+ self._families: dict[str, dict] = {}
+
+ def add(self, name, value, labels=None, help="", type="gauge") -> None:
+ if value is None:
+ return
+ fam = self._families.setdefault(name, {"help": help, "type": type,
+ "samples": []})
+ fam["samples"].append((labels or {}, value))
+
+ def render(self) -> str:
+ out = []
+ for name in sorted(self._families):
+ fam = self._families[name]
+ if not fam["samples"]:
+ continue
+ out.append(f"# HELP {name} {fam['help']}")
+ out.append(f"# TYPE {name} {fam['type']}")
+ for labels, value in fam["samples"]:
+ if isinstance(value, float):
+ rendered = f"{value:.6f}".rstrip("0").rstrip(".") or "0"
+ else:
+ rendered = str(value)
+ out.append(f"{name}{render_labels(labels)} {rendered}")
+ return "\n".join(out) + "\n"
+
+
+def run_json(args: list[str]) -> dict | None:
+ """Run smartctl with JSON output.
+
+ smartctl uses its exit status as a bitfield rather than a simple success flag --
+ bit 3 means "a self-test failed", which is data we specifically want, not an error.
+ So parse stdout regardless of exit status and only give up if there is no JSON.
+ """
+ try:
+ proc = subprocess.run(args, capture_output=True, text=True, timeout=120,
+ check=False)
+ except (OSError, subprocess.TimeoutExpired) as exc:
+ warn(f"{' '.join(args)}: {exc}")
+ return None
+ if not proc.stdout.strip():
+ err = proc.stderr.strip().splitlines()
+ warn(f"{' '.join(args)} produced no output: {err[0] if err else ''}")
+ return None
+ try:
+ return json.loads(proc.stdout)
+ except json.JSONDecodeError as exc:
+ warn(f"{' '.join(args)}: could not parse JSON: {exc}")
+ return None
+
+
+def scan_devices() -> list[str]:
+ data = run_json(["smartctl", "--scan", "-j"])
+ if not data:
+ return []
+ return [d["name"] for d in data.get("devices", []) if d.get("name")]
+
+
+def normalise_type(text: str) -> str:
+ for pattern, name in TEST_TYPES:
+ if pattern.search(text):
+ return name
+ return "other"
+
+
+def iter_entries(data: dict):
+ """Yield self-test log entries, whatever shape smartctl used.
+
+ ATA groups them into tables:
+ ata_smart_self_test_log: {standard: {table: [...]}, extended: {table: [...]}}
+
+ SCSI does not have a table at all -- smartctl emits one flat top-level key per
+ entry (scsiprint.cpp: `snprintf(st, "scsi_self_test_%d", k)`):
+ scsi_self_test_0: {code: {...}, result: {...}, power_on_time: {hours: N}}
+
+ Handling only the ATA shape would silently produce nothing for SAS drives.
+ """
+ for key, value in sorted(data.items()):
+ if "self_test" not in key or not isinstance(value, dict):
+ continue
+ # SCSI: the value *is* one entry.
+ if "code" in value or "result" in value:
+ yield value
+ continue
+ # ATA: a table here, or one level down under standard/extended.
+ if isinstance(value.get("table"), list):
+ yield from (e for e in value["table"] if isinstance(e, dict))
+ for sub in value.values():
+ if isinstance(sub, dict) and isinstance(sub.get("table"), list):
+ yield from (e for e in sub["table"] if isinstance(e, dict))
+
+
+def _field(entry: dict, *names):
+ for name in names:
+ if name in entry:
+ return entry[name]
+ return {}
+
+
+def entry_fields(entry: dict) -> tuple[str, str, bool | None, float | None]:
+ """Pull (test type, status text, passed, lifetime hours) out of a log entry.
+
+ All three transports name these differently:
+ ATA type / status{passed} / lifetime_hours
+ SCSI code / result / power_on_time{hours}
+ NVMe self_test_code / self_test_result / power_on_hours
+ """
+ type_field = _field(entry, "type", "code", "self_test_code")
+ if isinstance(type_field, dict):
+ type_text = type_field.get("string") or str(type_field.get("value", ""))
+ else:
+ type_text = str(type_field)
+
+ status = _field(entry, "status", "result", "self_test_result")
+ if isinstance(status, dict):
+ status_text = status.get("string") or str(status.get("value", ""))
+ passed = status.get("passed")
+ else:
+ status_text = str(status)
+ passed = None
+
+ if passed is None and status_text:
+ # SCSI logs carry no "passed" boolean, only a result string. Check for failure
+ # before checking for "Completed", because SCSI reports a failed long test as
+ # "Completed, segment failed" -- a naive startswith("completed") reads that as
+ # a pass.
+ low = status_text.lower()
+ if "in progress" in low or low.startswith("aborted"):
+ passed = None
+ elif "fail" in low or "error" in low:
+ passed = "without error" in low
+ elif low.startswith("completed"):
+ passed = True
+
+ hours = entry.get("lifetime_hours")
+ if hours is None and isinstance(entry.get("power_on_time"), dict):
+ hours = entry["power_on_time"].get("hours")
+ if hours is None and isinstance(entry.get("power_on_hours"), (int, float)):
+ hours = entry["power_on_hours"]
+
+ return normalise_type(type_text), status_text, passed, hours
+
+
+def human_capacity(nbytes) -> str:
+ """Format a capacity the way the label on the drive does: decimal, not binary."""
+ if not isinstance(nbytes, (int, float)) or nbytes <= 0:
+ return ""
+ for unit, scale in (("TB", 10 ** 12), ("GB", 10 ** 9), ("MB", 10 ** 6)):
+ if nbytes >= scale:
+ value = nbytes / scale
+ return f"{value:.1f} {unit}" if value < 100 else f"{value:.0f} {unit}"
+ return f"{int(nbytes)} B"
+
+
+def technology(data: dict) -> str:
+ """A short "what kind of drive is this" string: bus plus spindle speed."""
+ protocol = (data.get("device") or {}).get("protocol", "")
+ bus = {"ATA": "SATA", "SCSI": "SAS", "NVMe": "NVMe"}.get(protocol, protocol)
+
+ rpm = data.get("rotation_rate")
+ if rpm is None or rpm == 0:
+ # smartctl reports 0 (or omits it) for anything without platters.
+ return f"{bus} SSD".strip()
+ # 7200 -> "7.2K"; the round numbers people actually quote.
+ spin = f"{rpm / 1000:.1f}K".replace(".0K", "K")
+ return f"{bus} {spin}".strip()
+
+
+def collect_identity(reg: Registry, name: str, data: dict) -> None:
+ """Emit one info metric whose label is a preformatted identity line.
+
+ This exists to be read by a human in a Grafana row title, so it is deliberately a
+ single rendered string rather than separate labels -- Grafana can interpolate a
+ variable's display text into a title, but cannot assemble one from several labels.
+ """
+ model = (data.get("model_name") or data.get("scsi_model_name")
+ or (data.get("scsi_product") or "") or "unknown model")
+ serial = data.get("serial_number") or "no serial"
+ capacity = human_capacity((data.get("user_capacity") or {}).get("bytes"))
+
+ parts = [p for p in (capacity, technology(data), model, serial) if p]
+ reg.add("smart_drive_label", 1, {"device": name, "label": " · ".join(parts)},
+ "Drive identity rendered for display; the value is always 1.")
+
+
+def collect_device(reg: Registry, dev: str) -> bool:
+ # -i rides along on the same invocation, so identity costs no extra smartctl run
+ # and no extra chance of waking an idle disk.
+ data = run_json(["smartctl", "-j", "-i", "-l", "selftest", dev])
+ if data is None:
+ return False
+
+ name = os.path.basename(dev)
+ collect_identity(reg, name, data)
+ counts: dict[tuple[str, str], int] = defaultdict(int)
+ totals: dict[str, int] = defaultdict(int)
+ # smartctl lists newest first, so the first entry of each type is the latest.
+ latest: dict[str, tuple[str, bool | None, float | None]] = {}
+
+ entries = 0
+ for entry in iter_entries(data):
+ entries += 1
+ ttype, status_text, passed, hours = entry_fields(entry)
+ counts[(ttype, status_text)] += 1
+ totals[ttype] += 1
+ latest.setdefault(ttype, (status_text, passed, hours))
+
+ for (ttype, status_text), n in sorted(counts.items()):
+ reg.add("smart_selftest_result_count", n,
+ {"device": name, "test_type": ttype, "result": status_text},
+ "Self-test log entries by type and reported result.")
+
+ for ttype, n in sorted(totals.items()):
+ reg.add("smart_selftest_count", n, {"device": name, "test_type": ttype},
+ "Self-test log entries of this type.")
+
+ for ttype, (status_text, passed, hours) in sorted(latest.items()):
+ lbl = {"device": name, "test_type": ttype}
+ if passed is not None:
+ reg.add("smart_selftest_last_passed", int(bool(passed)), lbl,
+ "Whether the most recent self-test of this type passed.")
+ reg.add("smart_selftest_last_lifetime_hours", hours, lbl,
+ "Drive powered-on hours at which the most recent self-test of this "
+ "type ran. Subtract from the drive's current power-on hours to get "
+ "how long ago that was.")
+ reg.add("smart_selftest_last_status", 1, {**lbl, "status": status_text},
+ "Reported status text of the most recent self-test, 1 for the "
+ "current value.")
+
+ reg.add("smart_selftest_log_entries", entries, {"device": name},
+ "Total entries found in the drive's self-test log.")
+ return True
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--output", required=True,
+ help="path of the .prom file to write")
+ parser.add_argument("--device", action="append", default=[],
+ help="device to inspect; repeatable. Default: smartctl --scan")
+ args = parser.parse_args()
+
+ reg = Registry()
+ started = time.time()
+
+ devices = args.device or scan_devices()
+ if not devices:
+ warn("no devices found")
+
+ seen = 0
+ for dev in devices:
+ try:
+ if collect_device(reg, dev):
+ seen += 1
+ except Exception as exc: # noqa: BLE001 - one bad disk must not lose the rest
+ warn(f"{dev}: {exc!r}")
+
+ reg.add("smart_selftest_devices", seen, {},
+ "Devices successfully queried for self-test history.")
+ reg.add("smart_selftest_collection_duration_seconds", time.time() - started, {},
+ "Time this collector took to gather self-test history.")
+
+ tmp = f"{args.output}.tmp"
+ with open(tmp, "w") as handle:
+ handle.write(reg.render())
+ os.replace(tmp, args.output)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/collectors/zfs-metrics.py b/collectors/zfs-metrics.py
new file mode 100644
index 0000000..b313743
--- /dev/null
+++ b/collectors/zfs-metrics.py
@@ -0,0 +1,610 @@
+#!/usr/bin/env python3
+"""Collect ZFS metrics for the Prometheus node_exporter textfile collector.
+
+node_exporter's built-in zfs collector reads /proc/spl/kstat/zfs, which covers ARC
+statistics, per-dataset objset counters and pool state -- but nothing about capacity,
+fragmentation, vdev errors, scrub history, compression, encryption or snapshots. All of
+that has to come from the zpool/zfs CLIs, which is what this script does.
+
+Design notes:
+
+* Each collection stage is independent. A stage that fails logs to stderr and simply
+ contributes no samples, rather than taking the whole file down with it -- a broken
+ snapshot listing should not cost you your pool capacity graphs.
+* Output is written to a temporary file and renamed into place, so node_exporter never
+ reads a half-written file.
+* Samples are grouped into metric families by a registry rather than printed inline, so
+ the exposition format is correct no matter what order stages emit things in.
+
+Run it with --output PATH. Optionally pass --retention label=count (repeatable) to
+publish the configured auto-snapshot retention targets alongside the observed counts.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+import time
+from collections import defaultdict
+
+# zpool/zfs report state as a string; dashboards want something orderable.
+STATE_CODES = {
+ "ONLINE": 0,
+ "DEGRADED": 1,
+ "FAULTED": 2,
+ "OFFLINE": 3,
+ "UNAVAIL": 4,
+ "REMOVED": 5,
+ "SUSPENDED": 6,
+ "AVAIL": 7,
+ "INUSE": 7,
+}
+STATE_UNKNOWN = 8
+
+# zfs-auto-snapshot (zfstools) names snapshots "zfs-auto-snap_<label>-<Y-m-d-HhM>".
+AUTOSNAP_RE = re.compile(r"^zfs-auto-snap_([A-Za-z0-9]+)-")
+
+# How stale a snapshot of each label is allowed to get before it counts as overdue.
+# Mirrors the systemd timers NixOS creates for services.zfs.autoSnapshot.
+LABEL_INTERVALS = {
+ "frequent": 15 * 60,
+ "hourly": 60 * 60,
+ "daily": 24 * 60 * 60,
+ "weekly": 7 * 24 * 60 * 60,
+ "monthly": 31 * 24 * 60 * 60,
+}
+
+
+def warn(msg: str) -> None:
+ print(f"zfs-metrics: {msg}", file=sys.stderr)
+
+
+class Registry:
+ """Collects samples and renders them in Prometheus text exposition format."""
+
+ def __init__(self) -> None:
+ self._families: dict[str, dict] = {}
+
+ def add(self, name, value, labels=None, help="", type="gauge") -> None:
+ if value is None:
+ return
+ fam = self._families.get(name)
+ if fam is None:
+ fam = {"help": help, "type": type, "samples": []}
+ self._families[name] = fam
+ fam["samples"].append((labels or {}, value))
+
+ def render(self) -> str:
+ out = []
+ for name in sorted(self._families):
+ fam = self._families[name]
+ if not fam["samples"]:
+ continue
+ out.append(f"# HELP {name} {fam['help']}")
+ out.append(f"# TYPE {name} {fam['type']}")
+ for labels, value in fam["samples"]:
+ out.append(f"{name}{render_labels(labels)} {render_value(value)}")
+ return "\n".join(out) + "\n"
+
+
+def render_labels(labels: dict) -> str:
+ if not labels:
+ return ""
+ # Pool, dataset and vdev names cannot contain quotes or backslashes, so stripping
+ # them is a no-op in practice while guaranteeing we never emit a malformed label.
+ parts = [f'{k}="{str(v).translate(_LABEL_STRIP)}"' for k, v in sorted(labels.items())]
+ return "{" + ",".join(parts) + "}"
+
+
+_LABEL_STRIP = str.maketrans("", "", '"\\\n')
+
+
+def render_value(value) -> str:
+ if isinstance(value, float):
+ if value != value or value in (float("inf"), float("-inf")):
+ return "NaN"
+ # Avoid scientific notation, which Prometheus accepts but is unpleasant to read,
+ # while still keeping enough precision for byte counts.
+ return f"{value:.6f}".rstrip("0").rstrip(".") or "0"
+ return str(value)
+
+
+def run(args: list[str]) -> str | None:
+ """Run a command, returning stdout, or None if it failed."""
+ try:
+ proc = subprocess.run(
+ args, capture_output=True, text=True, timeout=120, check=False
+ )
+ except (OSError, subprocess.TimeoutExpired) as exc:
+ warn(f"{args[0]} {args[1] if len(args) > 1 else ''}: {exc}")
+ return None
+ if proc.returncode != 0:
+ err = proc.stderr.strip().splitlines()
+ warn(f"{' '.join(args)} exited {proc.returncode}: {err[0] if err else ''}")
+ return None
+ return proc.stdout
+
+
+def num(value: str):
+ """Parse a zfs/zpool numeric field, tolerating '-' and human-formatted suffixes."""
+ if value in ("", "-", "none"):
+ return None
+ value = value.rstrip("%x")
+ try:
+ return float(value) if "." in value else int(value)
+ except ValueError:
+ return None
+
+
+_SIZE_SUFFIXES = {"B": 1}
+for _i, _s in enumerate("KMGTPEZ", start=1):
+ _SIZE_SUFFIXES[_s] = 1024**_i
+
+
+def nicebytes(value: str):
+ """Parse zfs_nicebytes output, e.g. '0B', '512B', '1.05G'."""
+ if not value or value == "-":
+ return None
+ match = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)\s*([BKMGTPEZ])?", value.strip())
+ if not match:
+ return None
+ return float(match.group(1)) * _SIZE_SUFFIXES.get(match.group(2) or "B", 1)
+
+
+def parse_ctime(value: str):
+ """Parse a ctime(3) timestamp into a Unix epoch.
+
+ zpool status prints scan timestamps with ctime(), whose format is fixed and
+ locale-independent ("Sun Aug 10 05:36:35 2025"), so this is safe to parse. The day
+ is space-padded, hence the whitespace normalisation.
+ """
+ value = " ".join(value.split())
+ try:
+ return int(time.mktime(time.strptime(value, "%a %b %d %H:%M:%S %Y")))
+ except (ValueError, OverflowError):
+ return None
+
+
+def parse_dhms(value: str):
+ """Parse secs_to_dhms output: 'HH:MM:SS' or 'N days HH:MM:SS'."""
+ match = re.fullmatch(r"(?:(\d+) days )?(\d+):(\d{2}):(\d{2})", value.strip())
+ if not match:
+ return None
+ days, hours, mins, secs = match.groups()
+ return int(days or 0) * 86400 + int(hours) * 3600 + int(mins) * 60 + int(secs)
+
+
+def split_dataset(name: str) -> tuple[str, str]:
+ return name.split("/", 1)[0], name
+
+
+# --------------------------------------------------------------------------------------
+# Stage: pool capacity and health
+# --------------------------------------------------------------------------------------
+
+POOL_COLUMNS = [
+ "name", "size", "allocated", "free", "checkpoint", "fragmentation",
+ "capacity", "dedupratio", "health", "freeing", "leaked",
+]
+
+
+def collect_pools(reg: Registry) -> None:
+ out = run(["zpool", "list", "-Hp", "-o", ",".join(POOL_COLUMNS)])
+ if out is None:
+ return
+ for line in out.splitlines():
+ fields = line.split("\t")
+ if len(fields) != len(POOL_COLUMNS):
+ continue
+ row = dict(zip(POOL_COLUMNS, fields))
+ pool = row["name"]
+ lbl = {"pool": pool}
+
+ reg.add("zfs_pool_size_bytes", num(row["size"]), lbl,
+ "Total size of the pool.")
+ reg.add("zfs_pool_allocated_bytes", num(row["allocated"]), lbl,
+ "Space allocated in the pool.")
+ reg.add("zfs_pool_free_bytes", num(row["free"]), lbl,
+ "Space free in the pool.")
+ reg.add("zfs_pool_checkpoint_bytes", num(row["checkpoint"]), lbl,
+ "Space consumed by a pool checkpoint.")
+ reg.add("zfs_pool_freeing_bytes", num(row["freeing"]), lbl,
+ "Space still to be reclaimed in the background.")
+ reg.add("zfs_pool_leaked_bytes", num(row["leaked"]), lbl,
+ "Space leaked, which should normally be zero.")
+ reg.add("zfs_pool_dedup_ratio", num(row["dedupratio"]), lbl,
+ "Deduplication ratio.")
+
+ frag = num(row["fragmentation"])
+ if frag is not None:
+ reg.add("zfs_pool_fragmentation_ratio", frag / 100, lbl,
+ "Free space fragmentation, 0-1.")
+ cap = num(row["capacity"])
+ if cap is not None:
+ reg.add("zfs_pool_capacity_ratio", cap / 100, lbl,
+ "Allocated space as a fraction of pool size, 0-1.")
+
+ health = row["health"]
+ reg.add("zfs_pool_health", 1, {"pool": pool, "state": health},
+ "Pool health, 1 for the currently reported state.")
+ reg.add("zfs_pool_health_code", STATE_CODES.get(health, STATE_UNKNOWN), lbl,
+ "Pool health as a code: 0 ONLINE, 1 DEGRADED, 2 FAULTED, 3 OFFLINE, "
+ "4 UNAVAIL, 5 REMOVED, 6 SUSPENDED, 8 unknown.")
+
+
+# --------------------------------------------------------------------------------------
+# Stage: zpool status -- vdev topology, errors, scrub/resilver history
+# --------------------------------------------------------------------------------------
+
+CONFIG_HEADER_RE = re.compile(r"^\s*NAME\s+STATE\s+READ\s+WRITE\s+CKSUM")
+POOL_RE = re.compile(r"^\s*pool:\s*(\S+)")
+SCAN_RE = re.compile(r"^\s*scan:\s*(.*)$")
+ERRORS_RE = re.compile(r"^\s*errors:\s*(.*)$")
+
+SCAN_PATTERNS = [
+ ("scrub", "finished",
+ re.compile(r"^scrub repaired (\S+) in (.+?) with (\d+) errors on (.+)$")),
+ ("resilver", "finished",
+ re.compile(r"^resilvered (\S+) in (.+?) with (\d+) errors on (.+)$")),
+ ("scrub", "canceled", re.compile(r"^scrub canceled on (.+)$")),
+ ("resilver", "canceled", re.compile(r"^resilver canceled on (.+)$")),
+ ("scrub", "scanning", re.compile(r"^scrub in progress since (.+)$")),
+ ("scrub", "paused", re.compile(r"^scrub paused since (.+)$")),
+ ("resilver", "scanning", re.compile(r"^resilver in progress since (.+)$")),
+]
+
+
+def collect_status(reg: Registry) -> None:
+ out = run(["zpool", "status", "-p"])
+ if out is None:
+ return
+
+ pool = None
+ in_config = False
+ seen_scan: set[str] = set()
+ pools: list[str] = []
+
+ for line in out.splitlines():
+ match = POOL_RE.match(line)
+ if match:
+ pool = match.group(1)
+ pools.append(pool)
+ in_config = False
+ continue
+
+ if pool is None:
+ continue
+
+ match = SCAN_RE.match(line)
+ if match and pool not in seen_scan:
+ seen_scan.add(pool)
+ parse_scan(reg, pool, match.group(1).strip())
+ continue
+
+ match = ERRORS_RE.match(line)
+ if match:
+ in_config = False
+ reg.add("zfs_pool_data_errors", parse_data_errors(match.group(1)),
+ {"pool": pool},
+ "Data errors reported by zpool status; 0 means no known errors.")
+ continue
+
+ if CONFIG_HEADER_RE.match(line):
+ in_config = True
+ continue
+
+ if in_config:
+ if not line.strip():
+ in_config = False
+ continue
+ parse_vdev(reg, pool, line)
+
+ # A pool with no scan line at all has never been scrubbed; make that explicit
+ # rather than leaving the series absent.
+ for p in pools:
+ if p not in seen_scan:
+ set_scan_flags(reg, p, scrub=0, resilver=0)
+ reg.add("zfs_pool_scan_state", 1, {"pool": p, "state": "none"},
+ "Current scan state, 1 for the active state.")
+
+
+def parse_data_errors(text: str):
+ text = text.strip()
+ if not text:
+ return None
+ if "No known data errors" in text:
+ return 0
+ match = re.search(r"(\d+) data errors", text)
+ if match:
+ return int(match.group(1))
+ # e.g. "Permanent errors have been detected in the following files:" -- we know
+ # something is wrong but not how many, so report a non-zero count.
+ return 1
+
+
+def set_scan_flags(reg: Registry, pool: str, scrub: int, resilver: int) -> None:
+ reg.add("zfs_pool_scrub_in_progress", scrub, {"pool": pool},
+ "Whether a scrub is currently running.")
+ reg.add("zfs_pool_resilver_in_progress", resilver, {"pool": pool},
+ "Whether a resilver is currently running.")
+
+
+def parse_scan(reg: Registry, pool: str, text: str) -> None:
+ lbl = {"pool": pool}
+
+ for kind, state, pattern in SCAN_PATTERNS:
+ match = pattern.match(text)
+ if not match:
+ continue
+
+ scanning = state in ("scanning", "paused")
+ set_scan_flags(
+ reg, pool,
+ scrub=int(scanning and kind == "scrub"),
+ resilver=int(scanning and kind == "resilver"),
+ )
+ reg.add("zfs_pool_scan_state", 1, {"pool": pool, "state": f"{kind}_{state}"},
+ "Current scan state, 1 for the active state.")
+
+ if state == "finished":
+ repaired, duration, errors, when = match.groups()
+ reg.add(f"zfs_pool_last_{kind}_end_timestamp_seconds", parse_ctime(when),
+ lbl, f"Unix time the last {kind} completed.")
+ reg.add(f"zfs_pool_last_{kind}_duration_seconds", parse_dhms(duration),
+ lbl, f"How long the last completed {kind} took.")
+ reg.add(f"zfs_pool_last_{kind}_errors", int(errors), lbl,
+ f"Unrepairable errors found by the last completed {kind}.")
+ reg.add(f"zfs_pool_last_{kind}_repaired_bytes", nicebytes(repaired), lbl,
+ f"Bytes repaired by the last completed {kind}.")
+ elif state == "scanning":
+ reg.add(f"zfs_pool_{kind}_started_timestamp_seconds",
+ parse_ctime(match.group(1)), lbl,
+ f"Unix time the running {kind} started.")
+ return
+
+ # "none requested", or a format we do not recognise.
+ set_scan_flags(reg, pool, scrub=0, resilver=0)
+ state = "none" if "none requested" in text else "unknown"
+ reg.add("zfs_pool_scan_state", 1, {"pool": pool, "state": state},
+ "Current scan state, 1 for the active state.")
+
+
+def parse_vdev(reg: Registry, pool: str, line: str) -> None:
+ fields = line.split()
+ # vdev rows are "NAME STATE READ WRITE CKSUM" plus an optional trailing note.
+ # Section headers (logs/cache/spares) and spare rows have fewer fields.
+ if len(fields) < 5:
+ return
+ name, state, read, write, cksum = fields[:5]
+ if not (read.isdigit() and write.isdigit() and cksum.isdigit()):
+ return
+
+ reg.add("zfs_pool_vdev_state_code", STATE_CODES.get(state, STATE_UNKNOWN),
+ {"pool": pool, "vdev": name},
+ "vdev state as a code, same encoding as zfs_pool_health_code.")
+ for kind, value in (("read", read), ("write", write), ("checksum", cksum)):
+ reg.add("zfs_pool_vdev_errors_total", int(value),
+ {"pool": pool, "vdev": name, "type": kind},
+ "Cumulative vdev errors as reported by zpool status. "
+ "Reset by zpool clear.", type="counter")
+
+
+# --------------------------------------------------------------------------------------
+# Stage: datasets -- space, compression, encryption
+# --------------------------------------------------------------------------------------
+
+DATASET_COLUMNS = [
+ "name", "type", "used", "available", "referenced", "logicalused",
+ "usedbysnapshots", "usedbydataset", "compressratio", "compression",
+ "encryption", "keystatus", "quota", "refquota", "recordsize", "mounted",
+ "readonly", "com.sun:auto-snapshot",
+]
+
+ONOFF = {"on": 1, "off": 0, "yes": 1, "no": 0, "true": 1, "false": 0}
+
+# `zfs get keystatus` reports "none" for datasets that are not encrypted at all.
+KEYSTATUS_CODES = {"none": 0, "-": 0, "available": 1, "unavailable": 2}
+
+
+def collect_datasets(reg: Registry) -> None:
+ out = run([
+ "zfs", "list", "-Hp", "-t", "filesystem,volume",
+ "-o", ",".join(DATASET_COLUMNS),
+ ])
+ if out is None:
+ return
+
+ for line in out.splitlines():
+ fields = line.split("\t")
+ if len(fields) != len(DATASET_COLUMNS):
+ continue
+ row = dict(zip(DATASET_COLUMNS, fields))
+ pool, dataset = split_dataset(row["name"])
+ lbl = {"pool": pool, "dataset": dataset}
+
+ reg.add("zfs_dataset_used_bytes", num(row["used"]), lbl,
+ "Space consumed by the dataset and its descendants.")
+ reg.add("zfs_dataset_available_bytes", num(row["available"]), lbl,
+ "Space available to the dataset.")
+ reg.add("zfs_dataset_referenced_bytes", num(row["referenced"]), lbl,
+ "Space referenced by the dataset itself.")
+ reg.add("zfs_dataset_logical_used_bytes", num(row["logicalused"]), lbl,
+ "Logical (pre-compression) space consumed.")
+ reg.add("zfs_dataset_used_by_snapshots_bytes", num(row["usedbysnapshots"]), lbl,
+ "Space consumed by this dataset's snapshots.")
+ reg.add("zfs_dataset_used_by_dataset_bytes", num(row["usedbydataset"]), lbl,
+ "Space consumed by the dataset's own data.")
+ reg.add("zfs_dataset_compression_ratio", num(row["compressratio"]), lbl,
+ "Achieved compression ratio; 1 means no saving.")
+ reg.add("zfs_dataset_quota_bytes", num(row["quota"]), lbl,
+ "Configured quota in bytes; 0 means unlimited.")
+ reg.add("zfs_dataset_refquota_bytes", num(row["refquota"]), lbl,
+ "Configured refquota in bytes; 0 means unlimited.")
+ reg.add("zfs_dataset_recordsize_bytes", num(row["recordsize"]), lbl,
+ "Configured record size.")
+ reg.add("zfs_dataset_mounted", ONOFF.get(row["mounted"]), lbl,
+ "Whether the dataset is currently mounted.")
+ reg.add("zfs_dataset_readonly", ONOFF.get(row["readonly"]), lbl,
+ "Whether the dataset is read-only.")
+
+ autosnap = ONOFF.get(row["com.sun:auto-snapshot"])
+ reg.add("zfs_dataset_autosnapshot_enabled", autosnap, lbl,
+ "Whether com.sun:auto-snapshot is set true on the dataset.")
+
+ # Info-style metrics: the value is always 1, the detail lives in the labels.
+ # The label is named after the property rather than a generic "algorithm" so
+ # that dashboards can attach it to a numeric series with group_left() without
+ # two of these colliding on the same label name.
+ reg.add("zfs_dataset_compression", 1,
+ {**lbl, "compression": row["compression"]},
+ "Configured compression algorithm, 1 for the active setting.")
+ reg.add("zfs_dataset_encryption", 1,
+ {**lbl, "encryption": row["encryption"]},
+ "Configured encryption suite, 1 for the active setting.")
+ reg.add("zfs_dataset_encrypted", int(row["encryption"] not in ("off", "-")),
+ lbl, "Whether the dataset is encrypted.")
+ reg.add("zfs_dataset_key_available", int(row["keystatus"] == "available"),
+ lbl, "Whether the encryption key is loaded (unencrypted datasets "
+ "report 0).")
+ # keystatus is a three-state property -- "none" on an unencrypted dataset is
+ # not the same thing as "unavailable" on an encrypted one. Collapsing it to a
+ # boolean loses that distinction and makes every unencrypted dataset look
+ # locked, so publish the three states as a code too.
+ reg.add("zfs_dataset_keystatus_code", KEYSTATUS_CODES.get(row["keystatus"], 3),
+ lbl, "Encryption key status: 0 not encrypted, 1 key loaded, "
+ "2 key not loaded, 3 unknown.")
+ reg.add("zfs_dataset_type", 1, {**lbl, "type": row["type"]},
+ "Dataset type, 1 for the actual type.")
+
+
+# --------------------------------------------------------------------------------------
+# Stage: snapshots
+# --------------------------------------------------------------------------------------
+
+def collect_snapshots(reg: Registry) -> None:
+ out = run(["zfs", "list", "-Hp", "-t", "snapshot", "-o", "name,creation,used"])
+ if out is None:
+ return
+
+ # (dataset, label) -> [count, newest, oldest, used]
+ per_dataset: dict[tuple[str, str], list] = defaultdict(
+ lambda: [0, None, None, 0.0]
+ )
+ per_pool: dict[tuple[str, str], list] = defaultdict(lambda: [0, None, None, 0.0])
+
+ for line in out.splitlines():
+ fields = line.split("\t")
+ if len(fields) != 3:
+ continue
+ full, creation, used = fields
+ if "@" not in full:
+ continue
+ dataset, snapname = full.split("@", 1)
+ pool, _ = split_dataset(dataset)
+
+ created = num(creation)
+ if created is None:
+ continue
+ size = num(used) or 0
+
+ match = AUTOSNAP_RE.match(snapname)
+ label = match.group(1) if match else "manual"
+
+ for bucket, key in (
+ (per_dataset, (dataset, label)),
+ (per_pool, (pool, label)),
+ ):
+ entry = bucket[key]
+ entry[0] += 1
+ entry[1] = created if entry[1] is None else max(entry[1], created)
+ entry[2] = created if entry[2] is None else min(entry[2], created)
+ entry[3] += size
+
+ for (dataset, label), (count, newest, oldest, used) in per_dataset.items():
+ pool, _ = split_dataset(dataset)
+ lbl = {"pool": pool, "dataset": dataset, "label": label}
+ reg.add("zfs_snapshot_count", count, lbl,
+ "Number of snapshots of this dataset with this label.")
+ reg.add("zfs_snapshot_newest_timestamp_seconds", newest, lbl,
+ "Creation time of the most recent snapshot.")
+ reg.add("zfs_snapshot_oldest_timestamp_seconds", oldest, lbl,
+ "Creation time of the oldest snapshot; with the newest one this is "
+ "the retention window actually on disk.")
+ reg.add("zfs_snapshot_used_bytes", used, lbl,
+ "Space consumed by these snapshots.")
+
+ for (pool, label), (count, newest, oldest, used) in per_pool.items():
+ lbl = {"pool": pool, "label": label}
+ reg.add("zfs_pool_snapshot_count", count, lbl,
+ "Number of snapshots in the pool with this label.")
+ reg.add("zfs_pool_snapshot_newest_timestamp_seconds", newest, lbl,
+ "Creation time of the pool's most recent snapshot with this label.")
+ reg.add("zfs_pool_snapshot_oldest_timestamp_seconds", oldest, lbl,
+ "Creation time of the pool's oldest snapshot with this label.")
+ reg.add("zfs_pool_snapshot_used_bytes", used, lbl,
+ "Space consumed by the pool's snapshots with this label.")
+
+
+def collect_retention(reg: Registry, retention: dict[str, int]) -> None:
+ """Publish the configured auto-snapshot policy so dashboards can compare
+ observed snapshot counts and ages against what is supposed to exist."""
+ for label, keep in sorted(retention.items()):
+ reg.add("zfs_snapshot_retention_target", keep, {"label": label},
+ "Configured number of snapshots to keep for this label.")
+ for label, seconds in sorted(LABEL_INTERVALS.items()):
+ if label in retention:
+ reg.add("zfs_snapshot_interval_seconds", seconds, {"label": label},
+ "Expected interval between snapshots of this label.")
+
+
+# --------------------------------------------------------------------------------------
+
+
+def parse_retention(values: list[str]) -> dict[str, int]:
+ out = {}
+ for item in values:
+ label, _, count = item.partition("=")
+ try:
+ out[label] = int(count)
+ except ValueError:
+ warn(f"ignoring malformed --retention {item!r}")
+ return out
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--output", required=True,
+ help="path of the .prom file to write")
+ parser.add_argument("--retention", action="append", default=[], metavar="LABEL=N",
+ help="configured auto-snapshot retention, repeatable")
+ args = parser.parse_args()
+
+ reg = Registry()
+ started = time.time()
+
+ for stage in (collect_pools, collect_status, collect_datasets, collect_snapshots):
+ try:
+ stage(reg)
+ except Exception as exc: # noqa: BLE001 - one bad stage must not lose the rest
+ warn(f"{stage.__name__} failed: {exc!r}")
+
+ try:
+ collect_retention(reg, parse_retention(args.retention))
+ except Exception as exc: # noqa: BLE001
+ warn(f"collect_retention failed: {exc!r}")
+
+ reg.add("zfs_metrics_collection_duration_seconds", time.time() - started, {},
+ "Time this collector took to gather ZFS metrics.")
+
+ tmp = f"{args.output}.tmp"
+ with open(tmp, "w") as handle:
+ handle.write(reg.render())
+ os.replace(tmp, args.output)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/dashboards/gen_smart.py b/dashboards/gen_smart.py
new file mode 100644
index 0000000..2cc0568
--- /dev/null
+++ b/dashboards/gen_smart.py
@@ -0,0 +1,584 @@
+#!/usr/bin/env python3
+"""Generate the "Drive Health (SMART)" Grafana dashboard.
+
+Run this and commit the result:
+
+ python3 dashboards/gen_smart.py > dashboards/smart.json
+
+A CrystalDiskInfo-style view: a Good/Caution/Bad verdict per drive, the full raw SMART
+attribute table, and self-test history.
+
+The fleet is mixed, so there are two repeating sections:
+
+* SATA drives expose the classic ATA attribute table (value / worst / thresh / raw).
+* SAS drives expose none of it. Their health lives in SCSI error counters instead --
+ grown defect list, uncorrected errors, corrected-error breakdown.
+
+Drives are routed into the right section by *metric presence* rather than by the
+`protocol` label, which is more robust: only ATA drives have smartctl_device_attribute,
+and only SCSI drives have smartctl_scsi_grown_defect_list.
+
+Metrics used:
+
+* smartctl_* -- prometheus-community/smartctl_exporter
+* smartctl:health_code etc. -- recording rules in rules/smartctl.yml
+* smart_selftest_* -- collectors/smart-selftest.py
+"""
+
+import json
+
+from panels import (OK_BAD, dashboard, gp, query_var, row, stat, table,
+ timeseries)
+
+# Matches the verdict encoding in rules/smartctl.yml.
+HEALTH_MAP = [{
+ "type": "value",
+ "options": {
+ "0": {"text": "GOOD", "color": "green", "index": 0},
+ "1": {"text": "CAUTION", "color": "orange", "index": 1},
+ "2": {"text": "BAD", "color": "red", "index": 2},
+ },
+}]
+
+HEALTH_THRESHOLDS = {"mode": "absolute", "steps": [
+ {"color": "green", "value": None},
+ {"color": "orange", "value": 1},
+ {"color": "red", "value": 2},
+]}
+
+PASS_MAP = [{
+ "type": "value",
+ "options": {
+ "0": {"text": "FAILED", "color": "red", "index": 0},
+ "1": {"text": "passed", "color": "green", "index": 1},
+ },
+}]
+
+ATA = 'device=~"$ata_device"'
+SAS = 'device=~"$sas_device"'
+
+# Splits "sde · GOOD · 6.0 TB · SAS 7.2K · HGST HUS726060AL5210 · NAHPJS45" into a
+# selectable value (the leading device name) and display text (the whole line). The
+# groups nest rather than sit side by side, which is what lets text be a superset of
+# value; matching on whitespace keeps it independent of the separator the collector
+# chose.
+ROW_REGEX = r"/^(?<text>(?<value>\S+)\s.*)$/"
+
+
+def power_on(sel=""):
+ """Best available powered-on seconds per drive, in seconds.
+
+ On ATA, smartctl reads power_on_time straight from attribute 9 and it is reliable.
+ On SAS there is no such attribute: smartctl sources power_on_time from the
+ Background Scan Results log page, whose "Accumulated power on time" counter is
+ maintained by the background-scan subsystem rather than being the drive's lifetime
+ meter. On the HGST SAS drives here it reads a few thousand hours while the drives'
+ own self-test logs record tens of thousands.
+
+ The self-test log's lifetime-hours column is stamped by the drive at the moment
+ each test ran, so it can never exceed the true powered-on figure. Taking whichever
+ source is larger is therefore never an over-estimate, and recovers a sane number on
+ drives whose power-on counter under-counts or was reset.
+ """
+ braces = f"{{{sel}}}" if sel else ""
+ return (f'max by (device) (smartctl_device_power_on_seconds{braces} '
+ f'or (smart_selftest_last_lifetime_hours{braces} * 3600))')
+
+
+def drive_info(title, sel, pos, label_cols, renames, desc=""):
+ """One-row identity card for a drive: what it is, not how it is doing.
+
+ smartctl_device is an info metric whose value is always 1 and whose payload is
+ entirely in its labels, so the value column gets dropped. Rotation rate and
+ capacity are joined in as real numbers; the join is an outer one so an SSD, which
+ reports no rotation rate at all, still shows up with a blank RPM cell.
+ """
+ return table(
+ title, [
+ (f'smartctl_device{{{sel}}}', None),
+ (f'max by (device) (smartctl_device_rotation_rate{{{sel}}})', None),
+ (f'max by (device) (smartctl_device_capacity_bytes{{{sel}}})', None),
+ ],
+ pos,
+ keep="device|" + "|".join(label_cols) + "|Value.*",
+ renames={"__join__": "device", "device": "Device",
+ "Value #B": "RPM", "Value #C": "Capacity", **renames},
+ exclude=["Value #A"],
+ order=["device", *label_cols, "Value #B", "Value #C"],
+ overrides=[
+ {"matcher": {"id": "byName", "options": "Capacity"},
+ "properties": [{"id": "unit", "value": "bytes"}]},
+ {"matcher": {"id": "byName", "options": "RPM"},
+ "properties": [{"id": "unit", "value": "rotrpm"},
+ {"id": "noValue", "value": "SSD"}]},
+ ],
+ desc=desc,
+ )
+
+
+panels = []
+
+# ======================================================================================
+# Fleet Overview
+# ======================================================================================
+panels.append(row("Fleet Overview", 0))
+
+panels.append(stat(
+ "Healthy", [('count(smartctl:health_code == 0) or vector(0)', "drives")],
+ gp(0, 1, 4, 4), colormode="background",
+ thresholds={"mode": "absolute", "steps": [{"color": "green", "value": None}]},
+ desc="Drives with no failing attribute, no SCSI uncorrected errors and a passing "
+ "SMART self-assessment.",
+))
+panels.append(stat(
+ "Caution", [('count(smartctl:health_code == 1) or vector(0)', "drives")],
+ gp(4, 1, 4, 4), colormode="background",
+ thresholds={"mode": "absolute", "steps": [
+ {"color": "green", "value": None}, {"color": "orange", "value": 1}]},
+ desc="Reallocated / pending / uncorrectable sectors on SATA, or a non-empty grown "
+ "defect list on SAS. Not failing yet — worth watching the trend.",
+))
+panels.append(stat(
+ "Failing", [('count(smartctl:health_code == 2) or vector(0)', "drives")],
+ gp(8, 1, 4, 4), colormode="background", thresholds=OK_BAD,
+ desc="SMART self-assessment failed, an attribute is at or below its threshold, or "
+ "the drive reported errors it could not correct. Replace these.",
+))
+panels.append(stat(
+ "Drives", [('count(smartctl_device_smart_status)', "total")],
+ gp(12, 1, 4, 4),
+))
+panels.append(stat(
+ "Hottest Drive",
+ [('max(smartctl_device_temperature{temperature_type="current"})', "max")],
+ gp(16, 1, 4, 4), unit="celsius", colormode="background",
+ thresholds={"mode": "absolute", "steps": [
+ {"color": "green", "value": None},
+ {"color": "yellow", "value": 45},
+ {"color": "red", "value": 55},
+ ]},
+))
+panels.append(stat(
+ "Longest Powered On", [(f'max({power_on()})', "max")],
+ gp(20, 1, 4, 4), unit="s",
+))
+
+panels.append(table(
+ "All Drives", [
+ # group_left carries the descriptive labels from the info metric onto the
+ # verdict, so this one query supplies device, model, serial and protocol.
+ ('smartctl:health_code * on(device) group_left(model_name, serial_number, '
+ 'protocol) smartctl_device', None),
+ ('max by (device) (smartctl_device_temperature{temperature_type="current"})',
+ None),
+ (power_on(), None),
+ ('max by (device) (smartctl_device_capacity_bytes)', None),
+ ],
+ gp(0, 5, 16, 10), keep="device|model_name|serial_number|protocol|Value.*",
+ renames={
+ "__join__": "device", "device": "Device", "model_name": "Model",
+ "serial_number": "Serial", "protocol": "Bus",
+ "Value #A": "Health", "Value #B": "Temp",
+ "Value #C": "Powered On", "Value #D": "Capacity",
+ },
+ order=["device", "model_name", "serial_number", "protocol",
+ "Value #A", "Value #B", "Value #C", "Value #D"],
+ sort_by="Health",
+ overrides=[
+ {"matcher": {"id": "byName", "options": "Health"},
+ "properties": [
+ {"id": "mappings", "value": HEALTH_MAP},
+ {"id": "thresholds", "value": HEALTH_THRESHOLDS},
+ {"id": "custom.cellOptions",
+ "value": {"type": "color-background", "mode": "basic"}},
+ ]},
+ {"matcher": {"id": "byName", "options": "Temp"},
+ "properties": [{"id": "unit", "value": "celsius"}]},
+ {"matcher": {"id": "byName", "options": "Powered On"},
+ "properties": [{"id": "unit", "value": "s"}]},
+ {"matcher": {"id": "byName", "options": "Capacity"},
+ "properties": [{"id": "unit", "value": "bytes"}]},
+ ],
+ desc="Every drive, worst first. Health is computed by rules/smartctl.yml.",
+))
+
+panels.append(timeseries(
+ "Temperatures",
+ [('smartctl_device_temperature{temperature_type="current"}', "{{device}}")],
+ gp(16, 5, 8, 10), unit="celsius",
+ desc="Sustained operation above ~45C measurably shortens drive life.",
+))
+
+# ======================================================================================
+# SATA — repeats once per ATA drive
+# ======================================================================================
+panels.append(row("${ata_device:text}", 15, repeat="ata_device"))
+
+panels.append(stat(
+ "Health", [(f'smartctl:health_code{{{ATA}}}', "{{device}}")],
+ gp(0, 16, 6, 4), mappings=HEALTH_MAP, thresholds=HEALTH_THRESHOLDS,
+ colormode="background",
+))
+panels.append(stat(
+ "Temperature",
+ [(f'smartctl_device_temperature{{{ATA}, temperature_type="current"}}', "temp")],
+ gp(6, 16, 3, 4), unit="celsius",
+ thresholds={"mode": "absolute", "steps": [
+ {"color": "green", "value": None},
+ {"color": "yellow", "value": 45},
+ {"color": "red", "value": 55},
+ ]},
+))
+panels.append(stat(
+ "Powered On", [(power_on(ATA), "hours")],
+ gp(9, 16, 3, 4), unit="s",
+))
+panels.append(stat(
+ "Capacity", [(f'smartctl_device_capacity_bytes{{{ATA}}}', "bytes")],
+ gp(12, 16, 3, 4), unit="bytes",
+))
+panels.append(stat(
+ "Power Cycles", [(f'smartctl_device_power_cycle_count{{{ATA}}}', "cycles")],
+ gp(15, 16, 3, 4),
+))
+panels.append(stat(
+ "Reallocated Sectors",
+ [(f'smartctl_device_attribute{{{ATA}, attribute_id="5", '
+ f'attribute_value_type="raw"}}', "sectors")],
+ gp(18, 16, 3, 4), thresholds=OK_BAD, colormode="background",
+ desc="Attribute 5 raw. Sectors the drive has already retired. Any growth here is "
+ "the single most predictive sign of impending failure.",
+))
+panels.append(stat(
+ "Pending Sectors",
+ [(f'smartctl_device_attribute{{{ATA}, attribute_id="197", '
+ f'attribute_value_type="raw"}}', "sectors")],
+ gp(21, 16, 3, 4), thresholds=OK_BAD, colormode="background",
+ desc="Attribute 197 raw. Sectors that failed to read and are waiting to be "
+ "reallocated on next write. Often precedes reallocations.",
+))
+
+panels.append(drive_info(
+ "Drive — $ata_device", ATA, gp(0, 20, 24, 4),
+ label_cols=["model_family", "model_name", "serial_number", "firmware_version",
+ "interface", "sata_version", "form_factor"],
+ renames={"model_family": "Family", "model_name": "Model",
+ "serial_number": "Serial", "firmware_version": "Firmware",
+ "interface": "Interface", "sata_version": "SATA", "form_factor": "Form"},
+ desc="Identity of this drive. RPM shows SSD when the drive reports no rotation "
+ "rate.",
+))
+
+panels.append(table(
+ "SMART Attributes — $ata_device", [
+ # Only this query keeps the descriptive labels; the rest are reduced to the
+ # join key so nothing collides in joinByField.
+ (f'smartctl_device_attribute{{{ATA}, attribute_value_type="value"}}', None),
+ (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
+ f'attribute_value_type="worst"}})', None),
+ (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
+ f'attribute_value_type="thresh"}})', None),
+ (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
+ f'attribute_value_type="raw"}})', None),
+ # How much headroom is left before this attribute trips its own failure
+ # threshold. Zero or below is what SMART calls a failure.
+ (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
+ f'attribute_value_type="value"}} - on(device, attribute_id) '
+ f'smartctl_device_attribute{{{ATA}, attribute_value_type="thresh"}})', None),
+ ],
+ gp(0, 24, 24, 12), keep="attribute_id|attribute_name|Value.*",
+ renames={
+ "__join__": "attribute_id", "attribute_id": "ID",
+ "attribute_name": "Attribute",
+ "Value #A": "Current", "Value #B": "Worst", "Value #C": "Threshold",
+ "Value #D": "Raw", "Value #E": "Margin",
+ },
+ order=["attribute_id", "attribute_name",
+ "Value #A", "Value #B", "Value #C", "Value #D", "Value #E"],
+ sort_by="Attribute", sort_desc=False,
+ overrides=[
+ # Margin is the honest place to highlight trouble: a blanket threshold on Raw
+ # would flag Power_On_Hours, which is enormous and perfectly healthy.
+ {"matcher": {"id": "byName", "options": "Margin"},
+ "properties": [
+ {"id": "thresholds", "value": {"mode": "absolute", "steps": [
+ {"color": "red", "value": None},
+ {"color": "orange", "value": 1},
+ {"color": "yellow", "value": 10},
+ {"color": "green", "value": 25},
+ ]}},
+ {"id": "custom.cellOptions",
+ "value": {"type": "color-background", "mode": "basic"}},
+ ]},
+ ],
+ desc="Every attribute the drive reports, with its raw value. Current/Worst are "
+ "normalised 0-253 scores (higher is better); Threshold is where the vendor "
+ "declares failure; Margin is Current minus Threshold. ID sorts as text, so "
+ "the table is ordered by attribute name.",
+))
+
+# ======================================================================================
+# SAS — repeats once per SCSI drive
+# ======================================================================================
+panels.append(row("${sas_device:text}", 36, repeat="sas_device"))
+
+panels.append(stat(
+ "Health", [(f'smartctl:health_code{{{SAS}}}', "{{device}}")],
+ gp(0, 37, 6, 4), mappings=HEALTH_MAP, thresholds=HEALTH_THRESHOLDS,
+ colormode="background",
+))
+panels.append(stat(
+ "Temperature",
+ [(f'smartctl_device_temperature{{{SAS}, temperature_type="current"}}', "temp")],
+ gp(6, 37, 3, 4), unit="celsius",
+ thresholds={"mode": "absolute", "steps": [
+ {"color": "green", "value": None},
+ {"color": "yellow", "value": 45},
+ {"color": "red", "value": 55},
+ ]},
+))
+panels.append(stat(
+ "Powered On", [(power_on(SAS), "hours")],
+ gp(9, 37, 3, 4), unit="s",
+ desc="SAS drives have no ATA attribute 9, so smartctl reports power_on_time from "
+ "the Background Scan Results log page — a counter the background-scan "
+ "subsystem maintains, not the drive's lifetime meter, and one that commonly "
+ "reads far too low. This tile takes the larger of that counter and the "
+ "lifetime-hours stamp in the drive's own self-test log, which is why it can "
+ "exceed what `smartctl -x` prints as 'Accumulated power on time'.",
+))
+panels.append(stat(
+ "Capacity", [(f'smartctl_device_capacity_bytes{{{SAS}}}', "bytes")],
+ gp(12, 37, 3, 4), unit="bytes",
+))
+panels.append(stat(
+ "Link Speed",
+ [(f'smartctl_device_interface_speed{{{SAS}, speed_type="current"}} * 1000', "bps")],
+ gp(15, 37, 3, 4), unit="bps",
+ desc="Negotiated SAS link rate. A drive that has fallen back to a lower rate is "
+ "worth investigating — usually a cable or expander.",
+))
+panels.append(stat(
+ "Grown Defects", [(f'smartctl_scsi_grown_defect_list{{{SAS}}}', "defects")],
+ gp(18, 37, 3, 4), thresholds=OK_BAD, colormode="background",
+ desc="Sectors reallocated since the drive left the factory — the SAS equivalent "
+ "of ATA attribute 5. A stable non-zero count is tolerable; growth is not.",
+))
+panels.append(stat(
+ "Uncorrected Errors",
+ [(f'smartctl_read_total_uncorrected_errors{{{SAS}}} + '
+ f'smartctl_write_total_uncorrected_errors{{{SAS}}}', "errors")],
+ gp(21, 37, 3, 4), thresholds=OK_BAD, colormode="background",
+ desc="Errors the drive could not correct with ECC or retries, which means data "
+ "was lost. Should be zero.",
+))
+
+panels.append(drive_info(
+ "Drive — $sas_device", SAS, gp(0, 41, 24, 4),
+ label_cols=["scsi_vendor", "model_name", "serial_number", "scsi_revision",
+ "scsi_version", "protocol", "form_factor"],
+ renames={"scsi_vendor": "Vendor", "model_name": "Model",
+ "serial_number": "Serial", "scsi_revision": "Revision",
+ "scsi_version": "SCSI", "protocol": "Bus", "form_factor": "Form"},
+ desc="Identity of this drive. SAS drives leave firmware_version empty and report "
+ "their revision in scsi_revision instead.",
+))
+
+panels.append(timeseries(
+ "Corrected Errors — $sas_device", [
+ (f'smartctl_read_errors_corrected_by_eccfast{{{SAS}}}', "read ECC fast"),
+ (f'smartctl_read_errors_corrected_by_eccdelayed{{{SAS}}}', "read ECC delayed"),
+ (f'smartctl_read_errors_corrected_by_rereads_rewrites{{{SAS}}}', "read retries"),
+ (f'smartctl_write_errors_corrected_by_eccfast{{{SAS}}}', "write ECC fast"),
+ (f'smartctl_write_errors_corrected_by_eccdelayed{{{SAS}}}',
+ "write ECC delayed"),
+ (f'smartctl_write_errors_corrected_by_rereads_rewrites{{{SAS}}}',
+ "write retries"),
+ ],
+ gp(0, 45, 12, 8), minval=0,
+ desc="Errors the drive fixed on its own. ECC-fast corrections are routine and "
+ "grow constantly. Delayed corrections and especially rewrites/retries "
+ "climbing is the signal worth watching.",
+))
+panels.append(timeseries(
+ "Defects & Uncorrected — $sas_device", [
+ (f'smartctl_scsi_grown_defect_list{{{SAS}}}', "grown defects"),
+ (f'smartctl_read_total_uncorrected_errors{{{SAS}}}', "read uncorrected"),
+ (f'smartctl_write_total_uncorrected_errors{{{SAS}}}', "write uncorrected"),
+ ],
+ gp(12, 45, 12, 8), minval=0,
+ custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+ desc="Flat lines are good. Any upward step here is a real event.",
+))
+
+# ======================================================================================
+# Self-tests
+# ======================================================================================
+panels.append(row("Self-Tests", 53))
+
+panels.append(stat(
+ "Drives Never Tested",
+ [('count(smartctl_device_smart_status) - '
+ 'count(count by (device) (smart_selftest_last_lifetime_hours)) or '
+ 'count(smartctl_device_smart_status)', "drives")],
+ gp(0, 54, 4, 5), thresholds=OK_BAD, colormode="background",
+ desc="Drives with no self-test log entries at all. smartd only runs tests if its "
+ "config includes an -s schedule; merely enabling the service does not.",
+))
+panels.append(stat(
+ "Failed Self-Tests",
+ [('count(smart_selftest_last_passed == 0) or vector(0)', "tests")],
+ gp(4, 54, 4, 5), thresholds=OK_BAD, colormode="background",
+))
+panels.append(stat(
+ "Oldest Long Test",
+ [(f'max({power_on()} / 3600 - on(device) group_right() '
+ f'smart_selftest_last_lifetime_hours{{test_type="extended"}})', "hours ago")],
+ gp(8, 54, 4, 5), unit="h",
+ thresholds={"mode": "absolute", "steps": [
+ {"color": "green", "value": None},
+ {"color": "yellow", "value": 24 * 14},
+ {"color": "red", "value": 24 * 45},
+ ]},
+ no_value="never run",
+ desc="Powered-on hours since the least recently tested drive last completed an "
+ "extended (long) self-test. Shows 'never run' when no drive has ever "
+ "completed one — which is the default state, since smartd schedules no tests "
+ "unless its config carries an -s directive.",
+))
+
+def _keyed(expr):
+ """Reduce a series to just the synthetic join key.
+
+ joinByField takes a single field, but a row here is identified by device *and*
+ test_type together. Joining on either one alone silently mismatches rows across
+ drives, so build one composite label and join on that instead.
+ """
+ return (f'max by (drive_test) (label_join({expr}, "drive_test", "/", '
+ f'"device", "test_type"))')
+
+
+panels.append(table(
+ "Self-Test History", [
+ # Only this query keeps device and test_type as their own columns.
+ ('label_join(smart_selftest_last_passed, "drive_test", "/", "device", '
+ '"test_type")', None),
+ (_keyed('smart_selftest_last_lifetime_hours'), None),
+ (_keyed('smart_selftest_count'), None),
+ # Powered-on hours elapsed since that test ran. group_right() keeps the
+ # right-hand side's labels, so device and test_type survive the subtraction.
+ (_keyed(power_on() + ' / 3600 - on(device) group_right() '
+ 'smart_selftest_last_lifetime_hours'), None),
+ ],
+ gp(12, 54, 12, 10), keep="drive_test|device|test_type|Value.*",
+ renames={
+ "__join__": "drive_test", "device": "Device", "test_type": "Test",
+ "Value #A": "Last Result", "Value #B": "At Hours",
+ "Value #C": "Logged", "Value #D": "Hours Ago",
+ },
+ exclude=["drive_test"],
+ order=["device", "test_type", "Value #A", "Value #B", "Value #C", "Value #D"],
+ sort_by="Device", sort_desc=False,
+ overrides=[
+ {"matcher": {"id": "byName", "options": "Last Result"},
+ "properties": [
+ {"id": "mappings", "value": PASS_MAP},
+ {"id": "custom.cellOptions",
+ "value": {"type": "color-background", "mode": "basic"}},
+ ]},
+ ],
+ desc="Short and extended self-test results per drive, from "
+ "collectors/smart-selftest.py. 'At Hours' is the drive's powered-on hour "
+ "count when the test ran, straight from the log. 'Hours Ago' subtracts it "
+ "from the drive's current powered-on hours; on SAS that figure already falls "
+ "back to this log, so 0 means the newest log entry is also the highest hour "
+ "count the drive has ever reported — not that the test ran just now.",
+))
+
+# ======================================================================================
+# Collapsed: trends and diagnostics
+# ======================================================================================
+panels.append(row("Attribute Trends", 64, collapsed=True, children=[
+ timeseries(
+ "Reallocated Sectors (all SATA drives)",
+ [('smartctl_device_attribute{attribute_id="5", attribute_value_type="raw"}',
+ "{{device}}")],
+ gp(0, 65, 8, 7), minval=0,
+ custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+ ),
+ timeseries(
+ "Pending & Uncorrectable Sectors",
+ [('smartctl_device_attribute{attribute_id=~"197|198", '
+ 'attribute_value_type="raw"}', "{{device}} {{attribute_name}}")],
+ gp(8, 65, 8, 7), minval=0,
+ custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+ ),
+ timeseries(
+ "CRC Errors (cabling)",
+ [('smartctl_device_attribute{attribute_id="199", attribute_value_type="raw"}',
+ "{{device}}")],
+ gp(16, 65, 8, 7), minval=0,
+ custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+ desc="Attribute 199 counts errors on the SATA link rather than the platters. "
+ "Growth here usually means a cable or backplane, not a dying disk.",
+ ),
+ timeseries(
+ "Data Written / Read (SAS)", [
+ ('smartctl_device_bytes_written', "{{device}} written"),
+ ('smartctl_device_bytes_read', "{{device}} read"),
+ ],
+ gp(0, 72, 12, 7), unit="bytes", minval=0,
+ ),
+ timeseries(
+ "Health Code History",
+ [('smartctl:health_code', "{{device}}")],
+ gp(12, 72, 12, 7), mappings=HEALTH_MAP, minval=0, maxval=2,
+ custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"},
+ ),
+]))
+
+panels.append(row("Collector Health", 65, collapsed=True, children=[
+ timeseries(
+ "smartctl Exit Status",
+ [('smartctl_device_smartctl_exit_status', "{{device}}")],
+ gp(0, 66, 8, 7), minval=0,
+ desc="Non-zero means smartctl itself had trouble reading the device. Bit 0 is "
+ "a command-line problem, bit 1 a device-open failure, bit 3 a failing "
+ "drive.",
+ ),
+ timeseries(
+ "Self-Test Collector Duration",
+ [('smart_selftest_collection_duration_seconds', "collection")],
+ gp(8, 66, 8, 7), unit="s", minval=0,
+ ),
+ timeseries(
+ "Devices Seen", [
+ ('smartctl_devices', "exporter"),
+ ('smart_selftest_devices', "self-test collector"),
+ ],
+ gp(16, 66, 8, 7), minval=0,
+ desc="These should agree. A divergence means one of the two cannot see some "
+ "drives.",
+ ),
+]))
+
+print(json.dumps(dashboard(
+ title="Drive Health (SMART)",
+ uid="drive-health-smart",
+ description="Per-drive SMART health in the style of CrystalDiskInfo: a "
+ "Good/Caution/Bad verdict, full raw attribute tables for SATA, SCSI "
+ "error counters for SAS, and self-test history.",
+ panels=panels,
+ variables=[
+ # smartctl:row is one rendered line per drive, "sde · GOOD · 6.0 TB · ...".
+ # The nested capture groups keep the *value* a bare device name, so every
+ # `device=~"$ata_device"` selector below is unaffected, while the *text* the UI
+ # shows -- and that ${ata_device:text} puts in the row title -- is the whole
+ # line. That is what makes a collapsed row identifiable without opening it.
+ query_var("ata_device", 'label_values(smartctl:row{bus="ata"}, row)',
+ label="SATA drive", regex=ROW_REGEX),
+ query_var("sas_device", 'label_values(smartctl:row{bus="sas"}, row)',
+ label="SAS drive", regex=ROW_REGEX),
+ ],
+ tags=["smart", "storage", "hardware"],
+), indent=2))
diff --git a/dashboards/gen_zfs.py b/dashboards/gen_zfs.py
new file mode 100644
index 0000000..4edb38b
--- /dev/null
+++ b/dashboards/gen_zfs.py
@@ -0,0 +1,617 @@
+#!/usr/bin/env python3
+"""Generate the "ZFS Pools" Grafana dashboard.
+
+Run this and commit the result:
+
+ python3 dashboards/gen_zfs.py > dashboards/zfs-pools.json
+
+Hand-writing ~2000 lines of Grafana panel JSON invites subtle inconsistencies, so
+panels are built from a handful of helpers instead -- every timeseries goes through one
+code path and is therefore structurally identical by construction.
+
+The queries here are deliberately limited to metrics we actually produce:
+
+* zfs_pool_* / zfs_dataset_* / zfs_snapshot_* -- collectors/zfs-metrics.py
+* node_zfs_arc_* / node_zfs_zpool_dataset_* -- node_exporter's zfs collector
+* node_systemd_* -- node_exporter's systemd collector
+
+If you add a panel, add the metric to the collector first and keep this list honest.
+"""
+
+import json
+
+from panels import (DS, OK_BAD, bargauge, gauge, gp, row, stat, table, timeseries)
+
+
+STATE_MAP = [{
+ "type": "value",
+ "options": {
+ "0": {"text": "ONLINE", "color": "green", "index": 0},
+ "1": {"text": "DEGRADED", "color": "orange", "index": 1},
+ "2": {"text": "FAULTED", "color": "red", "index": 2},
+ "3": {"text": "OFFLINE", "color": "text", "index": 3},
+ "4": {"text": "UNAVAIL", "color": "red", "index": 4},
+ "5": {"text": "REMOVED", "color": "purple", "index": 5},
+ "6": {"text": "SUSPENDED", "color": "red", "index": 6},
+ "7": {"text": "AVAIL", "color": "blue", "index": 7},
+ "8": {"text": "UNKNOWN", "color": "text", "index": 8},
+ },
+}]
+
+BOOL_MAP = [{
+ "type": "value",
+ "options": {
+ "0": {"text": "idle", "color": "green", "index": 0},
+ "1": {"text": "RUNNING", "color": "blue", "index": 1},
+ },
+}]
+
+POOL = 'pool=~"$pool"'
+panels = []
+
+# ======================================================================================
+# Pool Overview
+# ======================================================================================
+panels.append(row("Pool Overview", 0))
+
+panels.append(stat(
+ "Pool Health", [(f'zfs_pool_health_code{{{POOL}}}', "{{pool}}")],
+ gp(0, 1, 4, 4), mappings=STATE_MAP, colormode="background",
+ desc="Health as reported by `zpool list -o health`.",
+))
+panels.append(stat(
+ "Data Errors", [(f'sum(zfs_pool_data_errors{{{POOL}}}) or vector(0)', "errors")],
+ gp(4, 1, 4, 4), thresholds=OK_BAD, colormode="background",
+ desc="Permanent, unrepairable errors from `zpool status`. Anything above zero "
+ "means data has been lost and the affected files need restoring.",
+))
+panels.append(stat(
+ "vdev Errors",
+ [(f'sum(zfs_pool_vdev_errors_total{{{POOL}}}) or vector(0)', "errors")],
+ gp(8, 1, 4, 4), thresholds=OK_BAD, colormode="background",
+ desc="Total read + write + checksum errors across all vdevs. Cleared by "
+ "`zpool clear`.",
+))
+panels.append(stat(
+ "Last Scrub",
+ [(f'time() - max(zfs_pool_last_scrub_end_timestamp_seconds{{{POOL}}})', "ago")],
+ gp(12, 1, 4, 4), unit="s", colormode="background",
+ thresholds={"mode": "absolute", "steps": [
+ {"color": "green", "value": None},
+ {"color": "yellow", "value": 35 * 86400},
+ {"color": "red", "value": 70 * 86400},
+ ]},
+ desc="Time since the most recent completed scrub. Turns amber past ~5 weeks, "
+ "which is a monthly scrub schedule having missed a run.",
+))
+panels.append(stat(
+ "Scrub / Resilver", [
+ (f'max(zfs_pool_scrub_in_progress{{{POOL}}})', "scrub"),
+ (f'max(zfs_pool_resilver_in_progress{{{POOL}}})', "resilver"),
+ ],
+ gp(16, 1, 4, 4), mappings=BOOL_MAP, colormode="background",
+))
+panels.append(stat(
+ "Locked Datasets",
+ [(f'count(zfs_dataset_encrypted{{{POOL}}} == 1 and '
+ f'zfs_dataset_key_available{{{POOL}}} == 0) or vector(0)', "locked")],
+ gp(20, 1, 4, 4), thresholds=OK_BAD, colormode="background",
+ desc="Encrypted datasets whose key is not loaded. These are unreadable and are "
+ "silently skipped by anything that walks the filesystem, including backups.",
+))
+
+panels.append(gauge(
+ "Space Used", [(f'zfs_pool_capacity_ratio{{{POOL}}}', "{{pool}}")],
+ gp(0, 5, 5, 4),
+ desc="How full the pool is: allocated / total. ZFS write performance degrades "
+ "sharply past ~80%, and the pool becomes effectively read-only near 100%. "
+ "The byte figures behind this are in 'Capacity by Pool' below.",
+))
+panels.append(gauge(
+ "Fragmentation", [(f'zfs_pool_fragmentation_ratio{{{POOL}}}', "{{pool}}")],
+ gp(5, 5, 5, 4),
+ thresholds={"mode": "absolute", "steps": [
+ {"color": "green", "value": None},
+ {"color": "yellow", "value": 0.5},
+ {"color": "red", "value": 0.75},
+ ]},
+ desc="Free-space fragmentation.",
+))
+panels.append(stat(
+ "Free Space", [(f'zfs_pool_free_bytes{{{POOL}}}', "{{pool}}")],
+ gp(10, 5, 5, 4), unit="bytes",
+))
+panels.append(stat(
+ "Snapshots Overdue",
+ [(f'count((time() - zfs_snapshot_newest_timestamp_seconds{{{POOL}}}) > on(label) '
+ f'group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)', "overdue")],
+ gp(15, 5, 4, 4), thresholds=OK_BAD, colormode="background",
+ desc="Dataset/label pairs whose newest snapshot is more than two intervals old — "
+ "i.e. the auto-snapshot timer has missed at least one run.",
+))
+panels.append(stat(
+ "Retention Depth",
+ [(f'time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{{{POOL}}})', "oldest")],
+ gp(19, 5, 5, 4), unit="s",
+ desc="Age of the oldest snapshot on the pool — how far back you can actually "
+ "restore from.",
+))
+
+panels.append(timeseries(
+ "Pool Space Used / Free", [
+ (f'zfs_pool_allocated_bytes{{{POOL}}}', "{{pool}} used"),
+ (f'zfs_pool_free_bytes{{{POOL}}}', "{{pool}} free"),
+ ],
+ gp(0, 9, 12, 8), unit="bytes", stack=True, minval=0,
+ desc="Stacked, so the top of each pool's band is its total size and the split "
+ "between the bands is the usage fraction the gauge above shows.",
+))
+panels.append(timeseries(
+ "Fragmentation",
+ [(f'zfs_pool_fragmentation_ratio{{{POOL}}}', "{{pool}}")],
+ gp(12, 9, 12, 8), unit="percentunit", minval=0, maxval=1,
+ desc="Free-space fragmentation — how scattered the pool's remaining free space "
+ "is, not how scattered the data is. It rises as a pool fills and stays high "
+ "afterwards; only rewriting data (send/recv, or deleting a lot) brings it "
+ "down. Kept on its own axis because it is unrelated to space used, and "
+ "plotting the two together invites reading one as the other.",
+))
+
+panels.append(table(
+ "Capacity by Pool", [
+ (f'zfs_pool_size_bytes{{{POOL}}}', None),
+ (f'max by (pool) (zfs_pool_allocated_bytes{{{POOL}}})', None),
+ (f'max by (pool) (zfs_pool_free_bytes{{{POOL}}})', None),
+ (f'max by (pool) (zfs_pool_capacity_ratio{{{POOL}}})', None),
+ (f'max by (pool) (zfs_pool_fragmentation_ratio{{{POOL}}})', None),
+ (f'max by (pool) (zfs_pool_snapshot_used_bytes{{{POOL}}})', None),
+ (f'max by (pool) (zfs_pool_dedup_ratio{{{POOL}}})', None),
+ ],
+ gp(0, 17, 24, 8), keep="pool|Value.*",
+ renames={
+ "__join__": "pool", "pool": "Pool",
+ "Value #A": "Size", "Value #B": "Used", "Value #C": "Free",
+ "Value #D": "Used %", "Value #E": "Fragmentation",
+ "Value #F": "Snapshots", "Value #G": "Dedup",
+ },
+ order=["pool", "Value #A", "Value #B", "Value #C", "Value #D", "Value #E",
+ "Value #F", "Value #G"],
+ sort_by="Used %",
+ overrides=[
+ # Anchored: byRegexp is a substring match, so a bare "Used" would also
+ # capture "Used %" and fight the percent override below.
+ {"matcher": {"id": "byRegexp", "options": "^(Size|Used|Free|Snapshots)$"},
+ "properties": [{"id": "unit", "value": "bytes"}]},
+ {"matcher": {"id": "byName", "options": "Used %"},
+ "properties": [
+ {"id": "unit", "value": "percentunit"},
+ {"id": "max", "value": 1},
+ {"id": "thresholds", "value": {"mode": "absolute", "steps": [
+ {"color": "green", "value": None},
+ {"color": "yellow", "value": 0.8},
+ {"color": "red", "value": 0.9},
+ ]}},
+ {"id": "custom.cellOptions",
+ "value": {"type": "gauge", "mode": "gradient"}},
+ ]},
+ {"matcher": {"id": "byName", "options": "Fragmentation"},
+ "properties": [{"id": "unit", "value": "percentunit"}]},
+ ],
+ desc="Per-pool space in bytes. 'Size' is the raw pool size zpool reports, which "
+ "for raidz is before parity overhead — so 'Free' is the number to trust for "
+ "how much you can still write. 'Snapshots' is space held only by snapshots.",
+))
+
+# ======================================================================================
+# Integrity & Errors
+# ======================================================================================
+panels.append(row("Integrity & Errors", 25))
+
+panels.append(table(
+ "vdev State", [(f'zfs_pool_vdev_state_code{{{POOL}}}', None)],
+ gp(0, 26, 12, 9), keep="pool|vdev|Value",
+ renames={"pool": "Pool", "vdev": "vdev", "Value": "State"},
+ mappings=STATE_MAP,
+ overrides=[{
+ "matcher": {"id": "byName", "options": "State"},
+ "properties": [{"id": "custom.cellOptions",
+ "value": {"type": "color-background", "mode": "basic"}}],
+ }],
+ desc="Every vdev and leaf device in the selected pool(s).",
+))
+panels.append(timeseries(
+ "vdev Error Counters",
+ [(f'zfs_pool_vdev_errors_total{{{POOL}}}', "{{pool}}/{{vdev}} {{type}}")],
+ gp(12, 26, 12, 9), minval=0,
+ custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+ desc="Cumulative per-vdev errors. A counter that climbs is a disk on its way out.",
+))
+
+panels.append(table(
+ "Scrub & Resilver History", [
+ (f'zfs_pool_last_scrub_end_timestamp_seconds{{{POOL}}} * 1000', None),
+ (f'zfs_pool_last_scrub_duration_seconds{{{POOL}}}', None),
+ (f'zfs_pool_last_scrub_errors{{{POOL}}}', None),
+ (f'zfs_pool_last_scrub_repaired_bytes{{{POOL}}}', None),
+ # zpool status reports one scan per pool, so a pool whose most recent scan was
+ # a resilver has no scrub record at all and would otherwise vanish from this
+ # table. The outer join keeps it, with only this column filled.
+ (f'zfs_pool_last_resilver_end_timestamp_seconds{{{POOL}}} * 1000', None),
+ ],
+ gp(0, 35, 12, 7), keep="pool|Value.*",
+ renames={
+ "__join__": "pool",
+ "pool": "Pool",
+ "Value #A": "Scrub Finished", "Value #B": "Duration",
+ "Value #C": "Errors", "Value #D": "Repaired",
+ "Value #E": "Resilver Finished",
+ },
+ overrides=[
+ {"matcher": {"id": "byName", "options": "Scrub Finished"},
+ "properties": [{"id": "unit", "value": "dateTimeAsIso"}]},
+ {"matcher": {"id": "byName", "options": "Resilver Finished"},
+ "properties": [{"id": "unit", "value": "dateTimeAsIso"}]},
+ {"matcher": {"id": "byName", "options": "Duration"},
+ "properties": [{"id": "unit", "value": "s"}]},
+ {"matcher": {"id": "byName", "options": "Repaired"},
+ "properties": [{"id": "unit", "value": "bytes"}]},
+ {"matcher": {"id": "byName", "options": "Errors"},
+ "properties": [
+ {"id": "custom.cellOptions",
+ "value": {"type": "color-background", "mode": "basic"}},
+ {"id": "thresholds", "value": OK_BAD},
+ ]},
+ ],
+ desc="Result of the last completed scrub per pool. Errors here are blocks that "
+ "could not be repaired from redundancy.",
+))
+panels.append(timeseries(
+ "Pool Data Errors", [(f'zfs_pool_data_errors{{{POOL}}}', "{{pool}}")],
+ gp(12, 35, 12, 7), minval=0,
+ custom={"lineWidth": 2, "fillOpacity": 10, "showPoints": "auto"},
+ desc="Permanent data errors over time.",
+))
+
+# ======================================================================================
+# Snapshots & Retention
+# ======================================================================================
+panels.append(row("Snapshots & Retention", 42))
+
+panels.append(bargauge(
+ "Time Since Last Snapshot",
+ [(f'time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{{{POOL}}})',
+ "{{label}}")],
+ gp(0, 43, 8, 8), unit="s",
+ thresholds={"mode": "absolute", "steps": [
+ {"color": "green", "value": None},
+ {"color": "yellow", "value": 2 * 3600},
+ {"color": "red", "value": 2 * 86400},
+ ]},
+ desc="Age of the newest snapshot of each label. Compare against the label's own "
+ "cadence — 'frequent' is every 15 minutes, 'monthly' every month, so the "
+ "colour thresholds here are only a coarse hint.",
+))
+panels.append(bargauge(
+ "Snapshots vs Retention Target",
+ [(f'min by (label) (zfs_snapshot_count{{{POOL}}} / on(label) group_left() '
+ f'zfs_snapshot_retention_target)', "{{label}}")],
+ gp(8, 43, 8, 8), unit="percentunit", maxv=1.2,
+ thresholds={"mode": "absolute", "steps": [
+ {"color": "red", "value": None},
+ {"color": "yellow", "value": 0.5},
+ {"color": "green", "value": 0.9},
+ ]},
+ desc="Worst dataset's snapshot count as a fraction of the configured "
+ "services.zfs.autoSnapshot retention. 100% means the full set is on disk; a "
+ "freshly-created dataset legitimately sits below target until it fills up.",
+))
+panels.append(bargauge(
+ "Oldest Snapshot",
+ [(f'time() - min by (label) '
+ f'(zfs_pool_snapshot_oldest_timestamp_seconds{{{POOL}}})', "{{label}}")],
+ gp(16, 43, 8, 8), unit="s",
+ desc="How far back each label reaches.",
+))
+
+panels.append(timeseries(
+ "Snapshot Count by Label",
+ [(f'sum by (label) (zfs_snapshot_count{{{POOL}}})', "{{label}}")],
+ gp(0, 51, 12, 8), minval=0,
+ desc="Total snapshots per label. A steady sawtooth is healthy rotation; a "
+ "monotonic decline means snapshots are expiring but not being replaced.",
+))
+panels.append(timeseries(
+ "Snapshot Space by Label",
+ [(f'sum by (label) (zfs_pool_snapshot_used_bytes{{{POOL}}})', "{{label}}")],
+ gp(12, 51, 12, 8), unit="bytes", stack=True, minval=0,
+ desc="Space held down by snapshots — the cost of your retention policy.",
+))
+
+panels.append(stat(
+ "Snapshot / Scrub Job Failures",
+ [('sum(node_systemd_unit_state{name=~"zfs-(snapshot|scrub).*",state="failed"}) '
+ 'or vector(0)', "failed units")],
+ gp(0, 59, 6, 6), thresholds=OK_BAD, colormode="background",
+ desc="systemd units for the auto-snapshot and auto-scrub timers currently in a "
+ "failed state.",
+))
+panels.append(bargauge(
+ "Time Since Timer Last Fired",
+ [('time() - node_systemd_timer_last_trigger_seconds{name=~"zfs-.*"}', "{{name}}")],
+ gp(6, 59, 18, 6), unit="s",
+ desc="When each zfs-* systemd timer last triggered. This is the job side of the "
+ "picture; the snapshot ages above are the result side.",
+))
+
+# ======================================================================================
+# Datasets
+# ======================================================================================
+panels.append(row("Datasets", 65))
+
+panels.append(timeseries(
+ "Dataset Usage Over Time",
+ [(f'zfs_dataset_used_bytes{{{POOL}}}', "{{dataset}}")],
+ gp(0, 66, 24, 9), unit="bytes", minval=0,
+ custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+ desc="Space each dataset is responsible for, including its snapshots and "
+ "descendants — so a parent's line contains its children's. Unstacked and "
+ "unfilled so a single dataset's growth is readable against the others rather "
+ "than riding on top of them. The legend carries last and max per dataset; the "
+ "instant breakdown is in 'Capacity by Dataset' below.",
+))
+
+panels.append(table(
+ "Capacity by Dataset", [
+ # Only this query keeps `pool` as its own column; the rest are reduced to the
+ # join key so joinByField has nothing to collide on. `dataset` is the full
+ # path ("ingens/media"), so it is unique on its own.
+ (f'zfs_dataset_used_bytes{{{POOL}}}', None),
+ (f'max by (dataset) (zfs_dataset_used_by_dataset_bytes{{{POOL}}})', None),
+ (f'max by (dataset) (zfs_dataset_used_by_snapshots_bytes{{{POOL}}})', None),
+ (f'max by (dataset) (zfs_dataset_referenced_bytes{{{POOL}}})', None),
+ (f'max by (dataset) (zfs_dataset_logical_used_bytes{{{POOL}}})', None),
+ (f'max by (dataset) (zfs_dataset_available_bytes{{{POOL}}})', None),
+ (f'max by (dataset) (zfs_dataset_compression_ratio{{{POOL}}})', None),
+ # Quota is 0 when unset, which would read as "no space allowed"; blank it.
+ (f'max by (dataset) (zfs_dataset_quota_bytes{{{POOL}}} > 0)', None),
+ ],
+ gp(0, 75, 24, 8), keep="pool|dataset|Value.*",
+ renames={
+ "__join__": "dataset", "dataset": "Dataset", "pool": "Pool",
+ "Value #A": "Used", "Value #B": "Own Data", "Value #C": "Snapshots",
+ "Value #D": "Referenced", "Value #E": "Logical", "Value #F": "Available",
+ "Value #G": "Ratio", "Value #H": "Quota",
+ },
+ order=["pool", "dataset", "Value #A", "Value #B", "Value #C", "Value #D",
+ "Value #E", "Value #F", "Value #G", "Value #H"],
+ sort_by="Used",
+ overrides=[
+ {"matcher": {"id": "byRegexp",
+ "options": "^(Used|Own Data|Snapshots|Referenced|Logical|"
+ "Available|Quota)$"},
+ "properties": [{"id": "unit", "value": "bytes"}]},
+ {"matcher": {"id": "byName", "options": "Quota"},
+ "properties": [{"id": "noValue", "value": "none"}]},
+ {"matcher": {"id": "byName", "options": "Ratio"},
+ "properties": [{"id": "decimals", "value": 2}]},
+ ],
+ desc="Per-dataset space, separate from the per-pool figures above. 'Used' is the "
+ "total a dataset is responsible for; 'Own Data' and 'Snapshots' are the two "
+ "halves of that. 'Logical' is the size before compression, so Logical / "
+ "Referenced is where the compression ratio comes from. 'Available' is shared "
+ "with every sibling dataset unless a quota says otherwise.",
+))
+
+panels.append(table(
+ "Compression",
+ [(f'zfs_dataset_compression_ratio{{{POOL}}} * on(pool, dataset) '
+ f'group_left(compression) zfs_dataset_compression{{{POOL}}}', None)],
+ gp(0, 83, 12, 8), keep="pool|dataset|compression|Value",
+ order=["pool", "dataset", "compression", "Value"],
+ renames={"pool": "Pool", "dataset": "Dataset",
+ "compression": "Algorithm", "Value": "Ratio"},
+ sort_by="Ratio",
+ desc="group_left() attaches the algorithm label from the info metric onto the "
+ "ratio, so this needs only one query and no join transformation.",
+))
+panels.append(table(
+ "Encryption",
+ [(f'zfs_dataset_keystatus_code{{{POOL}}} * on(pool, dataset) '
+ f'group_left(encryption) zfs_dataset_encryption{{{POOL}}}', None)],
+ gp(12, 83, 12, 8), keep="pool|dataset|encryption|Value",
+ order=["pool", "dataset", "encryption", "Value"],
+ renames={"pool": "Pool", "dataset": "Dataset",
+ "encryption": "Suite", "Value": "Key"},
+ mappings=[{"type": "value", "options": {
+ "0": {"text": "—", "color": "text", "index": 0},
+ "1": {"text": "unlocked", "color": "green", "index": 1},
+ "2": {"text": "LOCKED", "color": "red", "index": 2},
+ "3": {"text": "unknown", "color": "orange", "index": 3},
+ }}],
+ overrides=[{
+ "matcher": {"id": "byName", "options": "Key"},
+ "properties": [{"id": "custom.cellOptions",
+ "value": {"type": "color-background", "mode": "basic"}}],
+ }],
+ desc="Encryption suite per dataset and the state of its key. Datasets that are "
+ "not encrypted show suite 'off' and a dash — only an encrypted dataset can "
+ "be locked.",
+))
+
+# ======================================================================================
+# Performance
+# ======================================================================================
+panels.append(row("Performance", 91))
+
+panels.append(timeseries(
+ "Dataset Throughput", [
+ (f'rate(node_zfs_zpool_dataset_nread{{zpool=~"$pool"}}[$__rate_interval])',
+ "{{zpool}}/{{dataset}} read"),
+ (f'-rate(node_zfs_zpool_dataset_nwritten{{zpool=~"$pool"}}[$__rate_interval])',
+ "{{zpool}}/{{dataset}} write"),
+ ],
+ gp(0, 92, 12, 8), unit="Bps",
+ desc="Per-dataset throughput from node_exporter's objset kstats. Writes are drawn "
+ "negative.",
+))
+panels.append(timeseries(
+ "Dataset IOPS", [
+ (f'rate(node_zfs_zpool_dataset_reads{{zpool=~"$pool"}}[$__rate_interval])',
+ "{{zpool}}/{{dataset}} read"),
+ (f'-rate(node_zfs_zpool_dataset_writes{{zpool=~"$pool"}}[$__rate_interval])',
+ "{{zpool}}/{{dataset}} write"),
+ ],
+ gp(12, 92, 12, 8), unit="iops",
+ desc="Writes are drawn negative.",
+))
+
+panels.append(timeseries(
+ "ARC Size", [
+ ("node_zfs_arc_size", "size"),
+ ("node_zfs_arc_c_max", "target max (c_max)"),
+ ("node_zfs_arc_c_min", "target min (c_min)"),
+ ],
+ gp(0, 100, 8, 8), unit="bytes", minval=0,
+))
+panels.append(timeseries(
+ "ARC Hit Ratio",
+ [("rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min("
+ "rate(node_zfs_arc_hits[$__rate_interval]) + "
+ "rate(node_zfs_arc_misses[$__rate_interval]), 1)", "hit ratio")],
+ gp(8, 100, 8, 8), unit="percentunit", minval=0,
+ desc="clamp_min avoids a divide-by-zero while the pool is idle.",
+))
+panels.append(timeseries(
+ "ARC Hits / Misses", [
+ ("rate(node_zfs_arc_hits[$__rate_interval])", "hits"),
+ ("rate(node_zfs_arc_misses[$__rate_interval])", "misses"),
+ ],
+ gp(16, 100, 8, 8), unit="ops", minval=0,
+))
+
+# ======================================================================================
+# Collapsed: advanced / diagnostic
+# ======================================================================================
+panels.append(row("Pool Internals", 108, collapsed=True, children=[
+ timeseries(
+ "Reclaim & Leaks", [
+ (f'zfs_pool_freeing_bytes{{{POOL}}}', "{{pool}} freeing"),
+ (f'zfs_pool_leaked_bytes{{{POOL}}}', "{{pool}} leaked"),
+ (f'zfs_pool_checkpoint_bytes{{{POOL}}}', "{{pool}} checkpoint"),
+ ],
+ gp(0, 109, 8, 7), unit="bytes", minval=0,
+ desc="`freeing` is space still being reclaimed in the background after a "
+ "delete. `leaked` should always be zero.",
+ ),
+ timeseries(
+ "Dedup Ratio", [(f'zfs_pool_dedup_ratio{{{POOL}}}', "{{pool}}")],
+ gp(8, 109, 8, 7), minval=0,
+ desc="1.00 unless deduplication is enabled, which it usually should not be.",
+ ),
+ timeseries(
+ "Pool Health History", [
+ (f'zfs_pool_health_code{{{POOL}}}', "{{pool}}"),
+ ],
+ # Bounded to the state-code range so the y-axis reads as states, not 0-100.
+ gp(16, 109, 8, 7), mappings=STATE_MAP, minval=0, maxval=8,
+ custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"},
+ desc="Health code over time — useful for spotting a pool that briefly "
+ "degraded and recovered. zfs_pool_scan_state is also collected if you "
+ "want to plot scrub/resilver phases alongside it.",
+ ),
+ table(
+ "Dataset Properties", [
+ (f'zfs_dataset_recordsize_bytes{{{POOL}}}', None),
+ (f'zfs_dataset_quota_bytes{{{POOL}}}', None),
+ (f'zfs_dataset_refquota_bytes{{{POOL}}}', None),
+ (f'zfs_dataset_autosnapshot_enabled{{{POOL}}}', None),
+ ],
+ gp(0, 116, 24, 8), keep="dataset|Value.*",
+ renames={
+ "__join__": "dataset", "dataset": "Dataset",
+ "Value #A": "Record Size", "Value #B": "Quota",
+ "Value #C": "Ref Quota", "Value #D": "Auto-snapshot",
+ },
+ overrides=[
+ {"matcher": {"id": "byName", "options": "Record Size"},
+ "properties": [{"id": "unit", "value": "bytes"}]},
+ {"matcher": {"id": "byName", "options": "Quota"},
+ "properties": [{"id": "unit", "value": "bytes"}]},
+ {"matcher": {"id": "byName", "options": "Ref Quota"},
+ "properties": [{"id": "unit", "value": "bytes"}]},
+ {"matcher": {"id": "byName", "options": "Auto-snapshot"},
+ "properties": [{"id": "mappings", "value": [{"type": "value", "options": {
+ "0": {"text": "no", "color": "orange", "index": 0},
+ "1": {"text": "yes", "color": "green", "index": 1},
+ }}]}]},
+ ],
+ desc="A quota of 0 means unlimited. Auto-snapshot reflects the "
+ "com.sun:auto-snapshot property that zfs-auto-snapshot honours.",
+ ),
+]))
+
+panels.append(row("Collector Health", 109, collapsed=True, children=[
+ timeseries(
+ "ZFS Collector Duration",
+ [("zfs_metrics_collection_duration_seconds", "collection")],
+ gp(0, 110, 8, 7), unit="s", minval=0,
+ desc="How long collectors/zfs-metrics.py takes. Listing every snapshot is the "
+ "slow part; if this approaches the 1 minute timer interval, raise "
+ "OnUnitActiveSec.",
+ ),
+ timeseries(
+ "Textfile Age",
+ [("time() - node_textfile_mtime_seconds", "{{file}}")],
+ gp(8, 110, 8, 7), unit="s", minval=0,
+ desc="Age of each .prom file. If this climbs without bound the collector has "
+ "stopped writing and every ZFS metric above is stale.",
+ ),
+ timeseries(
+ "Scrape Health", [
+ ('up{job="node"}', "node exporter up"),
+ ("node_textfile_scrape_error", "textfile parse error"),
+ ],
+ gp(16, 110, 8, 7), minval=0,
+ desc="node_textfile_scrape_error goes to 1 if a .prom file is malformed, in "
+ "which case node_exporter drops it entirely.",
+ ),
+]))
+
+dashboard = {
+ "title": "ZFS Pools",
+ "uid": "zfs-pools",
+ "description": "ZFS capacity, health, integrity, snapshot retention, dataset "
+ "properties and performance. Fed by node_exporter's zfs and systemd "
+ "collectors plus collectors/zfs-metrics.py from lgtm.nix.",
+ "tags": ["zfs", "storage"],
+ "timezone": "browser",
+ "schemaVersion": 41,
+ "version": 1,
+ "editable": True,
+ "graphTooltip": 1,
+ "refresh": "1m",
+ "time": {"from": "now-24h", "to": "now"},
+ "templating": {"list": [{
+ "name": "pool",
+ "label": "Pool",
+ "type": "query",
+ "datasource": DS,
+ "query": {"qryType": 1, "query": "label_values(zfs_pool_health_code, pool)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"},
+ "refresh": 1,
+ "includeAll": True,
+ "allValue": ".*",
+ "multi": True,
+ "current": {"text": ["All"], "value": ["$__all"]},
+ "options": [],
+ "sort": 1,
+ }]},
+ "annotations": {"list": [{
+ "builtIn": 1, "datasource": {"type": "datasource", "uid": "grafana"},
+ "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts", "type": "dashboard",
+ }]},
+ "panels": panels,
+}
+
+print(json.dumps(dashboard, indent=2))
diff --git a/dashboards/panels.py b/dashboards/panels.py
new file mode 100644
index 0000000..faa60f7
--- /dev/null
+++ b/dashboards/panels.py
@@ -0,0 +1,291 @@
+"""Shared Grafana panel builders for the dashboard generators in this directory.
+
+Hand-writing Grafana panel JSON at any volume invites subtle inconsistencies -- one
+panel with a slightly different fieldConfig shape, a typo'd unit. Every panel here goes
+through one code path per type, so they are structurally identical by construction.
+
+Used by gen_zfs.py and gen_smart.py. Both are run as scripts from this directory, so a
+plain `from panels import ...` resolves.
+"""
+
+DS = {"type": "prometheus", "uid": "prometheus"}
+
+# Thresholds: green until the value reaches 1, then red. For "any of these is a problem"
+# counters.
+OK_BAD = {"mode": "absolute", "steps": [
+ {"color": "green", "value": None},
+ {"color": "red", "value": 1},
+]}
+
+PLAIN = {"mode": "absolute", "steps": [{"color": "green", "value": None}]}
+
+_id = [0]
+
+
+def nid():
+ _id[0] += 1
+ return _id[0]
+
+
+def target(expr, legend=None, instant=False, fmt=None, ref="A"):
+ t = {
+ "datasource": DS, "editorMode": "code", "expr": expr, "refId": ref,
+ "instant": instant, "range": not instant,
+ }
+ if legend is not None:
+ t["legendFormat"] = legend
+ if fmt:
+ t["format"] = fmt
+ return t
+
+
+def targets(specs, instant=False, fmt=None):
+ return [
+ target(expr, legend, instant=instant, fmt=fmt, ref=chr(ord("A") + i))
+ for i, (expr, legend) in enumerate(specs)
+ ]
+
+
+def gp(x, y, w, h):
+ return {"h": h, "w": w, "x": x, "y": y}
+
+
+def timeseries(title, specs, pos, unit=None, desc="", stack=False, minval=None,
+ maxval=None, custom=None, mappings=None):
+ cust = {
+ "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1,
+ "fillOpacity": 40 if stack else 15, "showPoints": "never",
+ "spanNulls": True, "axisBorderShow": False, "axisCenteredZero": False,
+ "gradientMode": "none", "pointSize": 5,
+ "scaleDistribution": {"type": "linear"},
+ "stacking": {"group": "A", "mode": "normal" if stack else "none"},
+ "thresholdsStyle": {"mode": "off"},
+ }
+ if custom:
+ cust.update(custom)
+ defaults = {
+ "color": {"mode": "palette-classic"},
+ "custom": cust,
+ "mappings": mappings or [],
+ "thresholds": PLAIN,
+ }
+ if unit:
+ defaults["unit"] = unit
+ if minval is not None:
+ defaults["min"] = minval
+ if maxval is not None:
+ defaults["max"] = maxval
+ return {
+ "type": "timeseries", "title": title, "description": desc,
+ "id": nid(), "datasource": DS, "gridPos": pos,
+ "fieldConfig": {"defaults": defaults, "overrides": []},
+ "options": {
+ "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table",
+ "placement": "bottom", "showLegend": True},
+ "tooltip": {"mode": "multi", "sort": "desc"},
+ },
+ "targets": targets(specs),
+ }
+
+
+def stat(title, specs, pos, unit=None, mappings=None, thresholds=None, desc="",
+ colormode="value", textmode="auto", graphmode="none", decimals=None,
+ no_value=None):
+ defaults = {
+ "color": {"mode": "thresholds"},
+ "mappings": mappings or [],
+ "thresholds": thresholds or PLAIN,
+ }
+ if unit:
+ defaults["unit"] = unit
+ if decimals is not None:
+ defaults["decimals"] = decimals
+ if no_value is not None:
+ # Beats a bare "No data" when an empty result is itself the answer.
+ defaults["noValue"] = no_value
+ return {
+ "type": "stat", "title": title, "description": desc,
+ "id": nid(), "datasource": DS, "gridPos": pos,
+ "fieldConfig": {"defaults": defaults, "overrides": []},
+ "options": {
+ "colorMode": colormode, "graphMode": graphmode, "justifyMode": "auto",
+ "orientation": "auto", "textMode": textmode, "wideLayout": True,
+ "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
+ },
+ "targets": targets(specs, instant=True),
+ }
+
+
+def gauge(title, specs, pos, unit="percentunit", thresholds=None, desc="", maxv=1):
+ return {
+ "type": "gauge", "title": title, "description": desc,
+ "id": nid(), "datasource": DS, "gridPos": pos,
+ "fieldConfig": {"defaults": {
+ "color": {"mode": "thresholds"},
+ "mappings": [], "min": 0, "max": maxv, "unit": unit,
+ "thresholds": thresholds or {"mode": "absolute", "steps": [
+ {"color": "green", "value": None},
+ {"color": "yellow", "value": 0.8},
+ {"color": "red", "value": 0.9},
+ ]},
+ }, "overrides": []},
+ "options": {
+ "showThresholdLabels": False, "showThresholdMarkers": True,
+ "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
+ },
+ "targets": targets(specs, instant=True),
+ }
+
+
+def bargauge(title, specs, pos, unit=None, thresholds=None, desc="", maxv=None,
+ display="basic"):
+ # "basic" draws one solid bar coloured by the value's own threshold. "lcd" splits
+ # the bar into segments coloured by position along the threshold scale, which makes
+ # a perfectly healthy value look mostly red — avoid it for anything with
+ # thresholds.
+ defaults = {
+ "color": {"mode": "thresholds"},
+ "mappings": [], "min": 0, "thresholds": thresholds or PLAIN,
+ }
+ if unit:
+ defaults["unit"] = unit
+ if maxv is not None:
+ defaults["max"] = maxv
+ return {
+ "type": "bargauge", "title": title, "description": desc,
+ "id": nid(), "datasource": DS, "gridPos": pos,
+ "fieldConfig": {"defaults": defaults, "overrides": []},
+ "options": {
+ "displayMode": display, "orientation": "horizontal",
+ "showUnfilled": True, "valueMode": "color",
+ "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
+ },
+ "targets": targets(specs, instant=True),
+ }
+
+
+def table(title, specs, pos, keep, renames, desc="", overrides=None, mappings=None,
+ sort_by=None, sort_desc=True, order=None, exclude=None):
+ """Instant-query table.
+
+ `keep` is a regex of the columns to retain, applied before the join. Filtering
+ first is what stops duplicated label columns (Time, __name__, job, instance) from
+ colliding and being silently renamed to "field 1"/"field 2" by joinByField.
+
+ When joining several queries, give exactly one of them the descriptive labels and
+ reduce the others with `max by (<join field>) (...)` so nothing collides.
+ """
+ transformations = [
+ {"id": "filterFieldsByName", "options": {"include": {"pattern": keep}}},
+ ]
+ if len(specs) > 1:
+ join_field = renames.get("__join__", "dataset")
+ transformations.append(
+ {"id": "joinByField", "options": {"byField": join_field, "mode": "outer"}}
+ )
+ transformations.append({"id": "organize", "options": {
+ # A synthetic join key built with label_join() has to survive `keep` so the
+ # join can use it, but should not be shown; drop it here.
+ "excludeByName": {name: True for name in (exclude or [])},
+ # indexByName keys are the *original* field names, before renameByName runs.
+ "indexByName": {name: i for i, name in enumerate(order or [])},
+ "renameByName": {k: v for k, v in renames.items() if k != "__join__"},
+ }})
+ if sort_by:
+ transformations.append(
+ {"id": "sortBy", "options": {"fields": {},
+ "sort": [{"field": sort_by,
+ "desc": sort_desc}]}}
+ )
+ return {
+ "type": "table", "title": title, "description": desc,
+ "id": nid(), "datasource": DS, "gridPos": pos,
+ "fieldConfig": {"defaults": {
+ "color": {"mode": "thresholds"},
+ "mappings": mappings or [],
+ "custom": {"align": "auto", "cellOptions": {"type": "auto"},
+ "inspect": False, "filterable": True},
+ "thresholds": PLAIN,
+ }, "overrides": overrides or []},
+ "options": {"showHeader": True, "cellHeight": "sm",
+ "footer": {"show": False, "reducer": ["sum"], "countRows": False,
+ "fields": ""}},
+ "targets": targets(specs, instant=True, fmt="table"),
+ "transformations": transformations,
+ }
+
+
+def row(title, y, collapsed=False, children=None, repeat=None):
+ """A dashboard row.
+
+ Grafana keeps a *collapsed* row's children nested inside the row panel, and an
+ *expanded* row's children at the top level. Getting that backwards makes panels
+ vanish, so callers pass `children` only for collapsed rows.
+
+ `repeat` names a template variable; Grafana then duplicates the row and everything
+ under it once per value of that variable.
+ """
+ r = {
+ "type": "row", "title": title, "id": nid(),
+ "gridPos": gp(0, y, 24, 1),
+ "collapsed": collapsed,
+ "panels": children or [],
+ }
+ if repeat:
+ r["repeat"] = repeat
+ r["repeatDirection"] = "h"
+ return r
+
+
+def query_var(name, query, label=None, multi=True, include_all=True, all_value=".*",
+ current_all=True, hide=0, regex=None):
+ """A Prometheus-backed template variable.
+
+ `regex` filters and rewrites the returned strings. Named capture groups `value` and
+ `text` split one string into what queries substitute and what the UI shows -- nest
+ them (`(?<text>(?<value>\\S+) .*)`) to show a superset of what you select on.
+ """
+ v = {
+ "name": name,
+ "label": label or name,
+ "type": "query",
+ "datasource": DS,
+ "query": {"qryType": 1, "query": query,
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"},
+ "refresh": 1,
+ "includeAll": include_all,
+ "multi": multi,
+ "options": [],
+ "sort": 1,
+ "hide": hide,
+ "regex": regex or "",
+ }
+ if include_all:
+ v["allValue"] = all_value
+ if current_all and include_all:
+ v["current"] = {"text": ["All"], "value": ["$__all"]}
+ return v
+
+
+def dashboard(title, uid, description, panels, variables, tags, refresh="1m",
+ time_from="now-24h"):
+ return {
+ "title": title,
+ "uid": uid,
+ "description": description,
+ "tags": tags,
+ "timezone": "browser",
+ "schemaVersion": 41,
+ "version": 1,
+ "editable": True,
+ "graphTooltip": 1,
+ "refresh": refresh,
+ "time": {"from": time_from, "to": "now"},
+ "templating": {"list": variables},
+ "annotations": {"list": [{
+ "builtIn": 1, "datasource": {"type": "datasource", "uid": "grafana"},
+ "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts", "type": "dashboard",
+ }]},
+ "panels": panels,
+ }
diff --git a/dashboards/smart.json b/dashboards/smart.json
new file mode 100644
index 0000000..ed48cd1
--- /dev/null
+++ b/dashboards/smart.json
@@ -0,0 +1,3844 @@
+{
+ "title": "Drive Health (SMART)",
+ "uid": "drive-health-smart",
+ "description": "Per-drive SMART health in the style of CrystalDiskInfo: a Good/Caution/Bad verdict, full raw attribute tables for SATA, SCSI error counters for SAS, and self-test history.",
+ "tags": [
+ "smart",
+ "storage",
+ "hardware"
+ ],
+ "timezone": "browser",
+ "schemaVersion": 41,
+ "version": 1,
+ "editable": true,
+ "graphTooltip": 1,
+ "refresh": "1m",
+ "time": {
+ "from": "now-24h",
+ "to": "now"
+ },
+ "templating": {
+ "list": [
+ {
+ "name": "ata_device",
+ "label": "SATA drive",
+ "type": "query",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "query": {
+ "qryType": 1,
+ "query": "label_values(smartctl:row{bus=\"ata\"}, row)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "refresh": 1,
+ "includeAll": true,
+ "multi": true,
+ "options": [],
+ "sort": 1,
+ "hide": 0,
+ "regex": "/^(?<text>(?<value>\\S+)\\s.*)$/",
+ "allValue": ".*",
+ "current": {
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ }
+ },
+ {
+ "name": "sas_device",
+ "label": "SAS drive",
+ "type": "query",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "query": {
+ "qryType": 1,
+ "query": "label_values(smartctl:row{bus=\"sas\"}, row)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "refresh": 1,
+ "includeAll": true,
+ "multi": true,
+ "options": [],
+ "sort": 1,
+ "hide": 0,
+ "regex": "/^(?<text>(?<value>\\S+)\\s.*)$/",
+ "allValue": ".*",
+ "current": {
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ }
+ }
+ ]
+ },
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "panels": [
+ {
+ "type": "row",
+ "title": "Fleet Overview",
+ "id": 1,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "collapsed": false,
+ "panels": []
+ },
+ {
+ "type": "stat",
+ "title": "Healthy",
+ "description": "Drives with no failing attribute, no SCSI uncorrected errors and a passing SMART self-assessment.",
+ "id": 2,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 0,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "count(smartctl:health_code == 0) or vector(0)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "drives"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Caution",
+ "description": "Reallocated / pending / uncorrectable sectors on SATA, or a non-empty grown defect list on SAS. Not failing yet \u2014 worth watching the trend.",
+ "id": 3,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 4,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "orange",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "count(smartctl:health_code == 1) or vector(0)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "drives"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Failing",
+ "description": "SMART self-assessment failed, an attribute is at or below its threshold, or the drive reported errors it could not correct. Replace these.",
+ "id": 4,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 8,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "count(smartctl:health_code == 2) or vector(0)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "drives"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Drives",
+ "description": "",
+ "id": 5,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 12,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "count(smartctl_device_smart_status)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "total"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Hottest Drive",
+ "description": "",
+ "id": 6,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 16,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 45
+ },
+ {
+ "color": "red",
+ "value": 55
+ }
+ ]
+ },
+ "unit": "celsius"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max(smartctl_device_temperature{temperature_type=\"current\"})",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "max"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Longest Powered On",
+ "description": "",
+ "id": 7,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 20,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)))",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "max"
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "title": "All Drives",
+ "description": "Every drive, worst first. Health is computed by rules/smartctl.yml.",
+ "id": 8,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 16,
+ "x": 0,
+ "y": 5
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Health"
+ },
+ "properties": [
+ {
+ "id": "mappings",
+ "value": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "GOOD",
+ "color": "green",
+ "index": 0
+ },
+ "1": {
+ "text": "CAUTION",
+ "color": "orange",
+ "index": 1
+ },
+ "2": {
+ "text": "BAD",
+ "color": "red",
+ "index": 2
+ }
+ }
+ }
+ ]
+ },
+ {
+ "id": "thresholds",
+ "value": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "orange",
+ "value": 1
+ },
+ {
+ "color": "red",
+ "value": 2
+ }
+ ]
+ }
+ },
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "type": "color-background",
+ "mode": "basic"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Temp"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "celsius"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Powered On"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "s"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Capacity"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl:health_code * on(device) group_left(model_name, serial_number, protocol) smartctl_device",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (device) (smartctl_device_temperature{temperature_type=\"current\"})",
+ "refId": "B",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600))",
+ "refId": "C",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (device) (smartctl_device_capacity_bytes)",
+ "refId": "D",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "device|model_name|serial_number|protocol|Value.*"
+ }
+ }
+ },
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "device",
+ "mode": "outer"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {},
+ "indexByName": {
+ "device": 0,
+ "model_name": 1,
+ "serial_number": 2,
+ "protocol": 3,
+ "Value #A": 4,
+ "Value #B": 5,
+ "Value #C": 6,
+ "Value #D": 7
+ },
+ "renameByName": {
+ "device": "Device",
+ "model_name": "Model",
+ "serial_number": "Serial",
+ "protocol": "Bus",
+ "Value #A": "Health",
+ "Value #B": "Temp",
+ "Value #C": "Powered On",
+ "Value #D": "Capacity"
+ }
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "fields": {},
+ "sort": [
+ {
+ "field": "Health",
+ "desc": true
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Temperatures",
+ "description": "Sustained operation above ~45C measurably shortens drive life.",
+ "id": 9,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 8,
+ "x": 16,
+ "y": 5
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "celsius"
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_temperature{temperature_type=\"current\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{device}}"
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "${ata_device:text}",
+ "id": 10,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 15
+ },
+ "collapsed": false,
+ "panels": [],
+ "repeat": "ata_device",
+ "repeatDirection": "h"
+ },
+ {
+ "type": "stat",
+ "title": "Health",
+ "description": "",
+ "id": 11,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 0,
+ "y": 16
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "GOOD",
+ "color": "green",
+ "index": 0
+ },
+ "1": {
+ "text": "CAUTION",
+ "color": "orange",
+ "index": 1
+ },
+ "2": {
+ "text": "BAD",
+ "color": "red",
+ "index": 2
+ }
+ }
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "orange",
+ "value": 1
+ },
+ {
+ "color": "red",
+ "value": 2
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl:health_code{device=~\"$ata_device\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "{{device}}"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Temperature",
+ "description": "",
+ "id": 12,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 6,
+ "y": 16
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 45
+ },
+ {
+ "color": "red",
+ "value": 55
+ }
+ ]
+ },
+ "unit": "celsius"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_temperature{device=~\"$ata_device\", temperature_type=\"current\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "temp"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Powered On",
+ "description": "",
+ "id": 13,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 9,
+ "y": 16
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$ata_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$ata_device\"} * 3600))",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "hours"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Capacity",
+ "description": "",
+ "id": 14,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 12,
+ "y": 16
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_capacity_bytes{device=~\"$ata_device\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "bytes"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Power Cycles",
+ "description": "",
+ "id": 15,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 15,
+ "y": 16
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_power_cycle_count{device=~\"$ata_device\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "cycles"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Reallocated Sectors",
+ "description": "Attribute 5 raw. Sectors the drive has already retired. Any growth here is the single most predictive sign of impending failure.",
+ "id": 16,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 18,
+ "y": 16
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"5\", attribute_value_type=\"raw\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "sectors"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Pending Sectors",
+ "description": "Attribute 197 raw. Sectors that failed to read and are waiting to be reallocated on next write. Often precedes reallocations.",
+ "id": 17,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 21,
+ "y": 16
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"197\", attribute_value_type=\"raw\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "sectors"
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "title": "Drive \u2014 $ata_device",
+ "description": "Identity of this drive. RPM shows SSD when the drive reports no rotation rate.",
+ "id": 18,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 24,
+ "x": 0,
+ "y": 20
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Capacity"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "RPM"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "rotrpm"
+ },
+ {
+ "id": "noValue",
+ "value": "SSD"
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device{device=~\"$ata_device\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$ata_device\"})",
+ "refId": "B",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$ata_device\"})",
+ "refId": "C",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "device|model_family|model_name|serial_number|firmware_version|interface|sata_version|form_factor|Value.*"
+ }
+ }
+ },
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "device",
+ "mode": "outer"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Value #A": true
+ },
+ "indexByName": {
+ "device": 0,
+ "model_family": 1,
+ "model_name": 2,
+ "serial_number": 3,
+ "firmware_version": 4,
+ "interface": 5,
+ "sata_version": 6,
+ "form_factor": 7,
+ "Value #B": 8,
+ "Value #C": 9
+ },
+ "renameByName": {
+ "device": "Device",
+ "Value #B": "RPM",
+ "Value #C": "Capacity",
+ "model_family": "Family",
+ "model_name": "Model",
+ "serial_number": "Serial",
+ "firmware_version": "Firmware",
+ "interface": "Interface",
+ "sata_version": "SATA",
+ "form_factor": "Form"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "title": "SMART Attributes \u2014 $ata_device",
+ "description": "Every attribute the drive reports, with its raw value. Current/Worst are normalised 0-253 scores (higher is better); Threshold is where the vendor declares failure; Margin is Current minus Threshold. ID sorts as text, so the table is ordered by attribute name.",
+ "id": 19,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 12,
+ "w": 24,
+ "x": 0,
+ "y": 24
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Margin"
+ },
+ "properties": [
+ {
+ "id": "thresholds",
+ "value": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ },
+ {
+ "color": "orange",
+ "value": 1
+ },
+ {
+ "color": "yellow",
+ "value": 10
+ },
+ {
+ "color": "green",
+ "value": 25
+ }
+ ]
+ }
+ },
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "type": "color-background",
+ "mode": "basic"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"worst\"})",
+ "refId": "B",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})",
+ "refId": "C",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"raw\"})",
+ "refId": "D",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"} - on(device, attribute_id) smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})",
+ "refId": "E",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "attribute_id|attribute_name|Value.*"
+ }
+ }
+ },
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "attribute_id",
+ "mode": "outer"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {},
+ "indexByName": {
+ "attribute_id": 0,
+ "attribute_name": 1,
+ "Value #A": 2,
+ "Value #B": 3,
+ "Value #C": 4,
+ "Value #D": 5,
+ "Value #E": 6
+ },
+ "renameByName": {
+ "attribute_id": "ID",
+ "attribute_name": "Attribute",
+ "Value #A": "Current",
+ "Value #B": "Worst",
+ "Value #C": "Threshold",
+ "Value #D": "Raw",
+ "Value #E": "Margin"
+ }
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "fields": {},
+ "sort": [
+ {
+ "field": "Attribute",
+ "desc": false
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "${sas_device:text}",
+ "id": 20,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 36
+ },
+ "collapsed": false,
+ "panels": [],
+ "repeat": "sas_device",
+ "repeatDirection": "h"
+ },
+ {
+ "type": "stat",
+ "title": "Health",
+ "description": "",
+ "id": 21,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 0,
+ "y": 37
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "GOOD",
+ "color": "green",
+ "index": 0
+ },
+ "1": {
+ "text": "CAUTION",
+ "color": "orange",
+ "index": 1
+ },
+ "2": {
+ "text": "BAD",
+ "color": "red",
+ "index": 2
+ }
+ }
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "orange",
+ "value": 1
+ },
+ {
+ "color": "red",
+ "value": 2
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl:health_code{device=~\"$sas_device\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "{{device}}"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Temperature",
+ "description": "",
+ "id": 22,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 6,
+ "y": 37
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 45
+ },
+ {
+ "color": "red",
+ "value": 55
+ }
+ ]
+ },
+ "unit": "celsius"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_temperature{device=~\"$sas_device\", temperature_type=\"current\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "temp"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Powered On",
+ "description": "SAS drives have no ATA attribute 9, so smartctl reports power_on_time from the Background Scan Results log page \u2014 a counter the background-scan subsystem maintains, not the drive's lifetime meter, and one that commonly reads far too low. This tile takes the larger of that counter and the lifetime-hours stamp in the drive's own self-test log, which is why it can exceed what `smartctl -x` prints as 'Accumulated power on time'.",
+ "id": 23,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 9,
+ "y": 37
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$sas_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$sas_device\"} * 3600))",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "hours"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Capacity",
+ "description": "",
+ "id": 24,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 12,
+ "y": 37
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_capacity_bytes{device=~\"$sas_device\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "bytes"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Link Speed",
+ "description": "Negotiated SAS link rate. A drive that has fallen back to a lower rate is worth investigating \u2014 usually a cable or expander.",
+ "id": 25,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 15,
+ "y": 37
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bps"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_interface_speed{device=~\"$sas_device\", speed_type=\"current\"} * 1000",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "bps"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Grown Defects",
+ "description": "Sectors reallocated since the drive left the factory \u2014 the SAS equivalent of ATA attribute 5. A stable non-zero count is tolerable; growth is not.",
+ "id": 26,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 18,
+ "y": 37
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "defects"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Uncorrected Errors",
+ "description": "Errors the drive could not correct with ECC or retries, which means data was lost. Should be zero.",
+ "id": 27,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 3,
+ "x": 21,
+ "y": 37
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"} + smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "errors"
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "title": "Drive \u2014 $sas_device",
+ "description": "Identity of this drive. SAS drives leave firmware_version empty and report their revision in scsi_revision instead.",
+ "id": 28,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 24,
+ "x": 0,
+ "y": 41
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Capacity"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "RPM"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "rotrpm"
+ },
+ {
+ "id": "noValue",
+ "value": "SSD"
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device{device=~\"$sas_device\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$sas_device\"})",
+ "refId": "B",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$sas_device\"})",
+ "refId": "C",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "device|scsi_vendor|model_name|serial_number|scsi_revision|scsi_version|protocol|form_factor|Value.*"
+ }
+ }
+ },
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "device",
+ "mode": "outer"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "Value #A": true
+ },
+ "indexByName": {
+ "device": 0,
+ "scsi_vendor": 1,
+ "model_name": 2,
+ "serial_number": 3,
+ "scsi_revision": 4,
+ "scsi_version": 5,
+ "protocol": 6,
+ "form_factor": 7,
+ "Value #B": 8,
+ "Value #C": 9
+ },
+ "renameByName": {
+ "device": "Device",
+ "Value #B": "RPM",
+ "Value #C": "Capacity",
+ "scsi_vendor": "Vendor",
+ "model_name": "Model",
+ "serial_number": "Serial",
+ "scsi_revision": "Revision",
+ "scsi_version": "SCSI",
+ "protocol": "Bus",
+ "form_factor": "Form"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Corrected Errors \u2014 $sas_device",
+ "description": "Errors the drive fixed on its own. ECC-fast corrections are routine and grow constantly. Delayed corrections and especially rewrites/retries climbing is the signal worth watching.",
+ "id": 29,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 45
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_read_errors_corrected_by_eccfast{device=~\"$sas_device\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "read ECC fast"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_read_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}",
+ "refId": "B",
+ "instant": false,
+ "range": true,
+ "legendFormat": "read ECC delayed"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_read_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}",
+ "refId": "C",
+ "instant": false,
+ "range": true,
+ "legendFormat": "read retries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_write_errors_corrected_by_eccfast{device=~\"$sas_device\"}",
+ "refId": "D",
+ "instant": false,
+ "range": true,
+ "legendFormat": "write ECC fast"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_write_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}",
+ "refId": "E",
+ "instant": false,
+ "range": true,
+ "legendFormat": "write ECC delayed"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_write_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}",
+ "refId": "F",
+ "instant": false,
+ "range": true,
+ "legendFormat": "write retries"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Defects & Uncorrected \u2014 $sas_device",
+ "description": "Flat lines are good. Any upward step here is a real event.",
+ "id": 30,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 45
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "fillOpacity": 0,
+ "showPoints": "auto",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "grown defects"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"}",
+ "refId": "B",
+ "instant": false,
+ "range": true,
+ "legendFormat": "read uncorrected"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}",
+ "refId": "C",
+ "instant": false,
+ "range": true,
+ "legendFormat": "write uncorrected"
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "Self-Tests",
+ "id": 31,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 53
+ },
+ "collapsed": false,
+ "panels": []
+ },
+ {
+ "type": "stat",
+ "title": "Drives Never Tested",
+ "description": "Drives with no self-test log entries at all. smartd only runs tests if its config includes an -s schedule; merely enabling the service does not.",
+ "id": 32,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 0,
+ "y": 54
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "count(smartctl_device_smart_status) - count(count by (device) (smart_selftest_last_lifetime_hours)) or count(smartctl_device_smart_status)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "drives"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Failed Self-Tests",
+ "description": "",
+ "id": 33,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 4,
+ "y": 54
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "count(smart_selftest_last_passed == 0) or vector(0)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "tests"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Oldest Long Test",
+ "description": "Powered-on hours since the least recently tested drive last completed an extended (long) self-test. Shows 'never run' when no drive has ever completed one \u2014 which is the default state, since smartd schedules no tests unless its config carries an -s directive.",
+ "id": 34,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 4,
+ "x": 8,
+ "y": 54
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 336
+ },
+ {
+ "color": "red",
+ "value": 1080
+ }
+ ]
+ },
+ "unit": "h",
+ "noValue": "never run"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{test_type=\"extended\"})",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "hours ago"
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "title": "Self-Test History",
+ "description": "Short and extended self-test results per drive, from collectors/smart-selftest.py. 'At Hours' is the drive's powered-on hour count when the test ran, straight from the log. 'Hours Ago' subtracts it from the drive's current powered-on hours; on SAS that figure already falls back to this log, so 0 means the newest log entry is also the highest hour count the drive has ever reported \u2014 not that the test ran just now.",
+ "id": 35,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 12,
+ "y": 54
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Last Result"
+ },
+ "properties": [
+ {
+ "id": "mappings",
+ "value": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "FAILED",
+ "color": "red",
+ "index": 0
+ },
+ "1": {
+ "text": "passed",
+ "color": "green",
+ "index": 1
+ }
+ }
+ }
+ ]
+ },
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "type": "color-background",
+ "mode": "basic"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "label_join(smart_selftest_last_passed, \"drive_test\", \"/\", \"device\", \"test_type\")",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (drive_test) (label_join(smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))",
+ "refId": "B",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (drive_test) (label_join(smart_selftest_count, \"drive_test\", \"/\", \"device\", \"test_type\"))",
+ "refId": "C",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (drive_test) (label_join(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))",
+ "refId": "D",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "drive_test|device|test_type|Value.*"
+ }
+ }
+ },
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "drive_test",
+ "mode": "outer"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {
+ "drive_test": true
+ },
+ "indexByName": {
+ "device": 0,
+ "test_type": 1,
+ "Value #A": 2,
+ "Value #B": 3,
+ "Value #C": 4,
+ "Value #D": 5
+ },
+ "renameByName": {
+ "device": "Device",
+ "test_type": "Test",
+ "Value #A": "Last Result",
+ "Value #B": "At Hours",
+ "Value #C": "Logged",
+ "Value #D": "Hours Ago"
+ }
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "fields": {},
+ "sort": [
+ {
+ "field": "Device",
+ "desc": false
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "Attribute Trends",
+ "id": 41,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 64
+ },
+ "collapsed": true,
+ "panels": [
+ {
+ "type": "timeseries",
+ "title": "Reallocated Sectors (all SATA drives)",
+ "description": "",
+ "id": 36,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 65
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "fillOpacity": 0,
+ "showPoints": "auto",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_attribute{attribute_id=\"5\", attribute_value_type=\"raw\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{device}}"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Pending & Uncorrectable Sectors",
+ "description": "",
+ "id": 37,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 65
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "fillOpacity": 0,
+ "showPoints": "auto",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_attribute{attribute_id=~\"197|198\", attribute_value_type=\"raw\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{device}} {{attribute_name}}"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "CRC Errors (cabling)",
+ "description": "Attribute 199 counts errors on the SATA link rather than the platters. Growth here usually means a cable or backplane, not a dying disk.",
+ "id": 38,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 65
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "fillOpacity": 0,
+ "showPoints": "auto",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_attribute{attribute_id=\"199\", attribute_value_type=\"raw\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{device}}"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Data Written / Read (SAS)",
+ "description": "",
+ "id": 39,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 72
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_bytes_written",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{device}} written"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_bytes_read",
+ "refId": "B",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{device}} read"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Health Code History",
+ "description": "",
+ "id": 40,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 72
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 2,
+ "fillOpacity": 0,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "GOOD",
+ "color": "green",
+ "index": 0
+ },
+ "1": {
+ "text": "CAUTION",
+ "color": "orange",
+ "index": 1
+ },
+ "2": {
+ "text": "BAD",
+ "color": "red",
+ "index": 2
+ }
+ }
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0,
+ "max": 2
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl:health_code",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{device}}"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "Collector Health",
+ "id": 45,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 65
+ },
+ "collapsed": true,
+ "panels": [
+ {
+ "type": "timeseries",
+ "title": "smartctl Exit Status",
+ "description": "Non-zero means smartctl itself had trouble reading the device. Bit 0 is a command-line problem, bit 1 a device-open failure, bit 3 a failing drive.",
+ "id": 42,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 66
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_device_smartctl_exit_status",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{device}}"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Self-Test Collector Duration",
+ "description": "",
+ "id": 43,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 66
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "s",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smart_selftest_collection_duration_seconds",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "collection"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Devices Seen",
+ "description": "These should agree. A divergence means one of the two cannot see some drives.",
+ "id": 44,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 66
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smartctl_devices",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "exporter"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "smart_selftest_devices",
+ "refId": "B",
+ "instant": false,
+ "range": true,
+ "legendFormat": "self-test collector"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
diff --git a/dashboards/zfs-pools.json b/dashboards/zfs-pools.json
new file mode 100644
index 0000000..0721bb8
--- /dev/null
+++ b/dashboards/zfs-pools.json
@@ -0,0 +1,4372 @@
+{
+ "title": "ZFS Pools",
+ "uid": "zfs-pools",
+ "description": "ZFS capacity, health, integrity, snapshot retention, dataset properties and performance. Fed by node_exporter's zfs and systemd collectors plus collectors/zfs-metrics.py from lgtm.nix.",
+ "tags": [
+ "zfs",
+ "storage"
+ ],
+ "timezone": "browser",
+ "schemaVersion": 41,
+ "version": 1,
+ "editable": true,
+ "graphTooltip": 1,
+ "refresh": "1m",
+ "time": {
+ "from": "now-24h",
+ "to": "now"
+ },
+ "templating": {
+ "list": [
+ {
+ "name": "pool",
+ "label": "Pool",
+ "type": "query",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "query": {
+ "qryType": 1,
+ "query": "label_values(zfs_pool_health_code, pool)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "refresh": 1,
+ "includeAll": true,
+ "allValue": ".*",
+ "multi": true,
+ "current": {
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "options": [],
+ "sort": 1
+ }
+ ]
+ },
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "panels": [
+ {
+ "type": "row",
+ "title": "Pool Overview",
+ "id": 1,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "collapsed": false,
+ "panels": []
+ },
+ {
+ "type": "stat",
+ "title": "Pool Health",
+ "description": "Health as reported by `zpool list -o health`.",
+ "id": 2,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 0,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "ONLINE",
+ "color": "green",
+ "index": 0
+ },
+ "1": {
+ "text": "DEGRADED",
+ "color": "orange",
+ "index": 1
+ },
+ "2": {
+ "text": "FAULTED",
+ "color": "red",
+ "index": 2
+ },
+ "3": {
+ "text": "OFFLINE",
+ "color": "text",
+ "index": 3
+ },
+ "4": {
+ "text": "UNAVAIL",
+ "color": "red",
+ "index": 4
+ },
+ "5": {
+ "text": "REMOVED",
+ "color": "purple",
+ "index": 5
+ },
+ "6": {
+ "text": "SUSPENDED",
+ "color": "red",
+ "index": 6
+ },
+ "7": {
+ "text": "AVAIL",
+ "color": "blue",
+ "index": 7
+ },
+ "8": {
+ "text": "UNKNOWN",
+ "color": "text",
+ "index": 8
+ }
+ }
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_health_code{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "{{pool}}"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Data Errors",
+ "description": "Permanent, unrepairable errors from `zpool status`. Anything above zero means data has been lost and the affected files need restoring.",
+ "id": 3,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 4,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum(zfs_pool_data_errors{pool=~\"$pool\"}) or vector(0)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "errors"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "vdev Errors",
+ "description": "Total read + write + checksum errors across all vdevs. Cleared by `zpool clear`.",
+ "id": 4,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 8,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum(zfs_pool_vdev_errors_total{pool=~\"$pool\"}) or vector(0)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "errors"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Last Scrub",
+ "description": "Time since the most recent completed scrub. Turns amber past ~5 weeks, which is a monthly scrub schedule having missed a run.",
+ "id": 5,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 12,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 3024000
+ },
+ {
+ "color": "red",
+ "value": 6048000
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "time() - max(zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"})",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "ago"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Scrub / Resilver",
+ "description": "",
+ "id": 6,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 16,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "idle",
+ "color": "green",
+ "index": 0
+ },
+ "1": {
+ "text": "RUNNING",
+ "color": "blue",
+ "index": 1
+ }
+ }
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max(zfs_pool_scrub_in_progress{pool=~\"$pool\"})",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "scrub"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max(zfs_pool_resilver_in_progress{pool=~\"$pool\"})",
+ "refId": "B",
+ "instant": true,
+ "range": false,
+ "legendFormat": "resilver"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Locked Datasets",
+ "description": "Encrypted datasets whose key is not loaded. These are unreadable and are silently skipped by anything that walks the filesystem, including backups.",
+ "id": 7,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 20,
+ "y": 1
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "count(zfs_dataset_encrypted{pool=~\"$pool\"} == 1 and zfs_dataset_key_available{pool=~\"$pool\"} == 0) or vector(0)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "locked"
+ }
+ ]
+ },
+ {
+ "type": "gauge",
+ "title": "Space Used",
+ "description": "How full the pool is: allocated / total. ZFS write performance degrades sharply past ~80%, and the pool becomes effectively read-only near 100%. The byte figures behind this are in 'Capacity by Pool' below.",
+ "id": 8,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 5,
+ "x": 0,
+ "y": 5
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "min": 0,
+ "max": 1,
+ "unit": "percentunit",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 0.8
+ },
+ {
+ "color": "red",
+ "value": 0.9
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_capacity_ratio{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "{{pool}}"
+ }
+ ]
+ },
+ {
+ "type": "gauge",
+ "title": "Fragmentation",
+ "description": "Free-space fragmentation.",
+ "id": 9,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 5,
+ "x": 5,
+ "y": 5
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "min": 0,
+ "max": 1,
+ "unit": "percentunit",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 0.5
+ },
+ {
+ "color": "red",
+ "value": 0.75
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "{{pool}}"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Free Space",
+ "description": "",
+ "id": 10,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 5,
+ "x": 10,
+ "y": 5
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "{{pool}}"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Snapshots Overdue",
+ "description": "Dataset/label pairs whose newest snapshot is more than two intervals old \u2014 i.e. the auto-snapshot timer has missed at least one run.",
+ "id": 11,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 15,
+ "y": 5
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "count((time() - zfs_snapshot_newest_timestamp_seconds{pool=~\"$pool\"}) > on(label) group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "overdue"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Retention Depth",
+ "description": "Age of the oldest snapshot on the pool \u2014 how far back you can actually restore from.",
+ "id": 12,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 5,
+ "x": 19,
+ "y": 5
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "oldest"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Pool Space Used / Free",
+ "description": "Stacked, so the top of each pool's band is its total size and the split between the bands is the usage fraction the gauge above shows.",
+ "id": 13,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 9
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 40,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_allocated_bytes{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{pool}} used"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}",
+ "refId": "B",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{pool}} free"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Fragmentation",
+ "description": "Free-space fragmentation \u2014 how scattered the pool's remaining free space is, not how scattered the data is. It rises as a pool fills and stays high afterwards; only rewriting data (send/recv, or deleting a lot) brings it down. Kept on its own axis because it is unrelated to space used, and plotting the two together invites reading one as the other.",
+ "id": 14,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 9
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "percentunit",
+ "min": 0,
+ "max": 1
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{pool}}"
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "title": "Capacity by Pool",
+ "description": "Per-pool space in bytes. 'Size' is the raw pool size zpool reports, which for raidz is before parity overhead \u2014 so 'Free' is the number to trust for how much you can still write. 'Snapshots' is space held only by snapshots.",
+ "id": 15,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 17
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "^(Size|Used|Free|Snapshots)$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Used %"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "percentunit"
+ },
+ {
+ "id": "max",
+ "value": 1
+ },
+ {
+ "id": "thresholds",
+ "value": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 0.8
+ },
+ {
+ "color": "red",
+ "value": 0.9
+ }
+ ]
+ }
+ },
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "type": "gauge",
+ "mode": "gradient"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Fragmentation"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "percentunit"
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (pool) (zfs_pool_allocated_bytes{pool=~\"$pool\"})",
+ "refId": "B",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (pool) (zfs_pool_free_bytes{pool=~\"$pool\"})",
+ "refId": "C",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (pool) (zfs_pool_capacity_ratio{pool=~\"$pool\"})",
+ "refId": "D",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (pool) (zfs_pool_fragmentation_ratio{pool=~\"$pool\"})",
+ "refId": "E",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (pool) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})",
+ "refId": "F",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (pool) (zfs_pool_dedup_ratio{pool=~\"$pool\"})",
+ "refId": "G",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "pool|Value.*"
+ }
+ }
+ },
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "pool",
+ "mode": "outer"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {},
+ "indexByName": {
+ "pool": 0,
+ "Value #A": 1,
+ "Value #B": 2,
+ "Value #C": 3,
+ "Value #D": 4,
+ "Value #E": 5,
+ "Value #F": 6,
+ "Value #G": 7
+ },
+ "renameByName": {
+ "pool": "Pool",
+ "Value #A": "Size",
+ "Value #B": "Used",
+ "Value #C": "Free",
+ "Value #D": "Used %",
+ "Value #E": "Fragmentation",
+ "Value #F": "Snapshots",
+ "Value #G": "Dedup"
+ }
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "fields": {},
+ "sort": [
+ {
+ "field": "Used %",
+ "desc": true
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "Integrity & Errors",
+ "id": 16,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 25
+ },
+ "collapsed": false,
+ "panels": []
+ },
+ {
+ "type": "table",
+ "title": "vdev State",
+ "description": "Every vdev and leaf device in the selected pool(s).",
+ "id": 17,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 26
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "ONLINE",
+ "color": "green",
+ "index": 0
+ },
+ "1": {
+ "text": "DEGRADED",
+ "color": "orange",
+ "index": 1
+ },
+ "2": {
+ "text": "FAULTED",
+ "color": "red",
+ "index": 2
+ },
+ "3": {
+ "text": "OFFLINE",
+ "color": "text",
+ "index": 3
+ },
+ "4": {
+ "text": "UNAVAIL",
+ "color": "red",
+ "index": 4
+ },
+ "5": {
+ "text": "REMOVED",
+ "color": "purple",
+ "index": 5
+ },
+ "6": {
+ "text": "SUSPENDED",
+ "color": "red",
+ "index": 6
+ },
+ "7": {
+ "text": "AVAIL",
+ "color": "blue",
+ "index": 7
+ },
+ "8": {
+ "text": "UNKNOWN",
+ "color": "text",
+ "index": 8
+ }
+ }
+ }
+ ],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "State"
+ },
+ "properties": [
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "type": "color-background",
+ "mode": "basic"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_vdev_state_code{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "pool|vdev|Value"
+ }
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {},
+ "indexByName": {},
+ "renameByName": {
+ "pool": "Pool",
+ "vdev": "vdev",
+ "Value": "State"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "vdev Error Counters",
+ "description": "Cumulative per-vdev errors. A counter that climbs is a disk on its way out.",
+ "id": 18,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 26
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "fillOpacity": 0,
+ "showPoints": "auto",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_vdev_errors_total{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{pool}}/{{vdev}} {{type}}"
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "title": "Scrub & Resilver History",
+ "description": "Result of the last completed scrub per pool. Errors here are blocks that could not be repaired from redundancy.",
+ "id": 19,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 35
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Scrub Finished"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "dateTimeAsIso"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Resilver Finished"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "dateTimeAsIso"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Duration"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "s"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Repaired"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Errors"
+ },
+ "properties": [
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "type": "color-background",
+ "mode": "basic"
+ }
+ },
+ {
+ "id": "thresholds",
+ "value": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"} * 1000",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_last_scrub_duration_seconds{pool=~\"$pool\"}",
+ "refId": "B",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_last_scrub_errors{pool=~\"$pool\"}",
+ "refId": "C",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_last_scrub_repaired_bytes{pool=~\"$pool\"}",
+ "refId": "D",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_last_resilver_end_timestamp_seconds{pool=~\"$pool\"} * 1000",
+ "refId": "E",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "pool|Value.*"
+ }
+ }
+ },
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "pool",
+ "mode": "outer"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {},
+ "indexByName": {},
+ "renameByName": {
+ "pool": "Pool",
+ "Value #A": "Scrub Finished",
+ "Value #B": "Duration",
+ "Value #C": "Errors",
+ "Value #D": "Repaired",
+ "Value #E": "Resilver Finished"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Pool Data Errors",
+ "description": "Permanent data errors over time.",
+ "id": 20,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 35
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "fillOpacity": 10,
+ "showPoints": "auto",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_data_errors{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{pool}}"
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "Snapshots & Retention",
+ "id": 21,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 42
+ },
+ "collapsed": false,
+ "panels": []
+ },
+ {
+ "type": "bargauge",
+ "title": "Time Since Last Snapshot",
+ "description": "Age of the newest snapshot of each label. Compare against the label's own cadence \u2014 'frequent' is every 15 minutes, 'monthly' every month, so the colour thresholds here are only a coarse hint.",
+ "id": 22,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 43
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 7200
+ },
+ {
+ "color": "red",
+ "value": 172800
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "options": {
+ "displayMode": "basic",
+ "orientation": "horizontal",
+ "showUnfilled": true,
+ "valueMode": "color",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{pool=~\"$pool\"})",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "{{label}}"
+ }
+ ]
+ },
+ {
+ "type": "bargauge",
+ "title": "Snapshots vs Retention Target",
+ "description": "Worst dataset's snapshot count as a fraction of the configured services.zfs.autoSnapshot retention. 100% means the full set is on disk; a freshly-created dataset legitimately sits below target until it fills up.",
+ "id": 23,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 43
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": null
+ },
+ {
+ "color": "yellow",
+ "value": 0.5
+ },
+ {
+ "color": "green",
+ "value": 0.9
+ }
+ ]
+ },
+ "unit": "percentunit",
+ "max": 1.2
+ },
+ "overrides": []
+ },
+ "options": {
+ "displayMode": "basic",
+ "orientation": "horizontal",
+ "showUnfilled": true,
+ "valueMode": "color",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "min by (label) (zfs_snapshot_count{pool=~\"$pool\"} / on(label) group_left() zfs_snapshot_retention_target)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "{{label}}"
+ }
+ ]
+ },
+ {
+ "type": "bargauge",
+ "title": "Oldest Snapshot",
+ "description": "How far back each label reaches.",
+ "id": 24,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 43
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "options": {
+ "displayMode": "basic",
+ "orientation": "horizontal",
+ "showUnfilled": true,
+ "valueMode": "color",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "time() - min by (label) (zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "{{label}}"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Snapshot Count by Label",
+ "description": "Total snapshots per label. A steady sawtooth is healthy rotation; a monotonic decline means snapshots are expiring but not being replaced.",
+ "id": 25,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 51
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (label) (zfs_snapshot_count{pool=~\"$pool\"})",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{label}}"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Snapshot Space by Label",
+ "description": "Space held down by snapshots \u2014 the cost of your retention policy.",
+ "id": 26,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 51
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 40,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (label) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{label}}"
+ }
+ ]
+ },
+ {
+ "type": "stat",
+ "title": "Snapshot / Scrub Job Failures",
+ "description": "systemd units for the auto-snapshot and auto-scrub timers currently in a failed state.",
+ "id": 27,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 59
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "textMode": "auto",
+ "wideLayout": true,
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum(node_systemd_unit_state{name=~\"zfs-(snapshot|scrub).*\",state=\"failed\"}) or vector(0)",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "failed units"
+ }
+ ]
+ },
+ {
+ "type": "bargauge",
+ "title": "Time Since Timer Last Fired",
+ "description": "When each zfs-* systemd timer last triggered. This is the job side of the picture; the snapshot ages above are the result side.",
+ "id": 28,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 18,
+ "x": 6,
+ "y": 59
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "options": {
+ "displayMode": "basic",
+ "orientation": "horizontal",
+ "showUnfilled": true,
+ "valueMode": "color",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "time() - node_systemd_timer_last_trigger_seconds{name=~\"zfs-.*\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "legendFormat": "{{name}}"
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "Datasets",
+ "id": 29,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 65
+ },
+ "collapsed": false,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Dataset Usage Over Time",
+ "description": "Space each dataset is responsible for, including its snapshots and descendants \u2014 so a parent's line contains its children's. Unstacked and unfilled so a single dataset's growth is readable against the others rather than riding on top of them. The legend carries last and max per dataset; the instant breakdown is in 'Capacity by Dataset' below.",
+ "id": 30,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 66
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 2,
+ "fillOpacity": 0,
+ "showPoints": "auto",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{dataset}}"
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "title": "Capacity by Dataset",
+ "description": "Per-dataset space, separate from the per-pool figures above. 'Used' is the total a dataset is responsible for; 'Own Data' and 'Snapshots' are the two halves of that. 'Logical' is the size before compression, so Logical / Referenced is where the compression ratio comes from. 'Available' is shared with every sibling dataset unless a quota says otherwise.",
+ "id": 31,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 75
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": "^(Used|Own Data|Snapshots|Referenced|Logical|Available|Quota)$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Quota"
+ },
+ "properties": [
+ {
+ "id": "noValue",
+ "value": "none"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Ratio"
+ },
+ "properties": [
+ {
+ "id": "decimals",
+ "value": 2
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (dataset) (zfs_dataset_used_by_dataset_bytes{pool=~\"$pool\"})",
+ "refId": "B",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (dataset) (zfs_dataset_used_by_snapshots_bytes{pool=~\"$pool\"})",
+ "refId": "C",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (dataset) (zfs_dataset_referenced_bytes{pool=~\"$pool\"})",
+ "refId": "D",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (dataset) (zfs_dataset_logical_used_bytes{pool=~\"$pool\"})",
+ "refId": "E",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (dataset) (zfs_dataset_available_bytes{pool=~\"$pool\"})",
+ "refId": "F",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (dataset) (zfs_dataset_compression_ratio{pool=~\"$pool\"})",
+ "refId": "G",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "max by (dataset) (zfs_dataset_quota_bytes{pool=~\"$pool\"} > 0)",
+ "refId": "H",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "pool|dataset|Value.*"
+ }
+ }
+ },
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "dataset",
+ "mode": "outer"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {},
+ "indexByName": {
+ "pool": 0,
+ "dataset": 1,
+ "Value #A": 2,
+ "Value #B": 3,
+ "Value #C": 4,
+ "Value #D": 5,
+ "Value #E": 6,
+ "Value #F": 7,
+ "Value #G": 8,
+ "Value #H": 9
+ },
+ "renameByName": {
+ "dataset": "Dataset",
+ "pool": "Pool",
+ "Value #A": "Used",
+ "Value #B": "Own Data",
+ "Value #C": "Snapshots",
+ "Value #D": "Referenced",
+ "Value #E": "Logical",
+ "Value #F": "Available",
+ "Value #G": "Ratio",
+ "Value #H": "Quota"
+ }
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "fields": {},
+ "sort": [
+ {
+ "field": "Used",
+ "desc": true
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "title": "Compression",
+ "description": "group_left() attaches the algorithm label from the info metric onto the ratio, so this needs only one query and no join transformation.",
+ "id": 32,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 83
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_dataset_compression_ratio{pool=~\"$pool\"} * on(pool, dataset) group_left(compression) zfs_dataset_compression{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "pool|dataset|compression|Value"
+ }
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {},
+ "indexByName": {
+ "pool": 0,
+ "dataset": 1,
+ "compression": 2,
+ "Value": 3
+ },
+ "renameByName": {
+ "pool": "Pool",
+ "dataset": "Dataset",
+ "compression": "Algorithm",
+ "Value": "Ratio"
+ }
+ }
+ },
+ {
+ "id": "sortBy",
+ "options": {
+ "fields": {},
+ "sort": [
+ {
+ "field": "Ratio",
+ "desc": true
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "title": "Encryption",
+ "description": "Encryption suite per dataset and the state of its key. Datasets that are not encrypted show suite 'off' and a dash \u2014 only an encrypted dataset can be locked.",
+ "id": 33,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 83
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "\u2014",
+ "color": "text",
+ "index": 0
+ },
+ "1": {
+ "text": "unlocked",
+ "color": "green",
+ "index": 1
+ },
+ "2": {
+ "text": "LOCKED",
+ "color": "red",
+ "index": 2
+ },
+ "3": {
+ "text": "unknown",
+ "color": "orange",
+ "index": 3
+ }
+ }
+ }
+ ],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Key"
+ },
+ "properties": [
+ {
+ "id": "custom.cellOptions",
+ "value": {
+ "type": "color-background",
+ "mode": "basic"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_dataset_keystatus_code{pool=~\"$pool\"} * on(pool, dataset) group_left(encryption) zfs_dataset_encryption{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "pool|dataset|encryption|Value"
+ }
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {},
+ "indexByName": {
+ "pool": 0,
+ "dataset": 1,
+ "encryption": 2,
+ "Value": 3
+ },
+ "renameByName": {
+ "pool": "Pool",
+ "dataset": "Dataset",
+ "encryption": "Suite",
+ "Value": "Key"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "Performance",
+ "id": 34,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 91
+ },
+ "collapsed": false,
+ "panels": []
+ },
+ {
+ "type": "timeseries",
+ "title": "Dataset Throughput",
+ "description": "Per-dataset throughput from node_exporter's objset kstats. Writes are drawn negative.",
+ "id": 35,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 92
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "rate(node_zfs_zpool_dataset_nread{zpool=~\"$pool\"}[$__rate_interval])",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{zpool}}/{{dataset}} read"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "-rate(node_zfs_zpool_dataset_nwritten{zpool=~\"$pool\"}[$__rate_interval])",
+ "refId": "B",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{zpool}}/{{dataset}} write"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Dataset IOPS",
+ "description": "Writes are drawn negative.",
+ "id": 36,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 92
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "iops"
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "rate(node_zfs_zpool_dataset_reads{zpool=~\"$pool\"}[$__rate_interval])",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{zpool}}/{{dataset}} read"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "-rate(node_zfs_zpool_dataset_writes{zpool=~\"$pool\"}[$__rate_interval])",
+ "refId": "B",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{zpool}}/{{dataset}} write"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "ARC Size",
+ "description": "",
+ "id": 37,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 100
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "node_zfs_arc_size",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "size"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "node_zfs_arc_c_max",
+ "refId": "B",
+ "instant": false,
+ "range": true,
+ "legendFormat": "target max (c_max)"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "node_zfs_arc_c_min",
+ "refId": "C",
+ "instant": false,
+ "range": true,
+ "legendFormat": "target min (c_min)"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "ARC Hit Ratio",
+ "description": "clamp_min avoids a divide-by-zero while the pool is idle.",
+ "id": 38,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 100
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "percentunit",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min(rate(node_zfs_arc_hits[$__rate_interval]) + rate(node_zfs_arc_misses[$__rate_interval]), 1)",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "hit ratio"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "ARC Hits / Misses",
+ "description": "",
+ "id": 39,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 100
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "ops",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "rate(node_zfs_arc_hits[$__rate_interval])",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "hits"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "rate(node_zfs_arc_misses[$__rate_interval])",
+ "refId": "B",
+ "instant": false,
+ "range": true,
+ "legendFormat": "misses"
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "Pool Internals",
+ "id": 44,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 108
+ },
+ "collapsed": true,
+ "panels": [
+ {
+ "type": "timeseries",
+ "title": "Reclaim & Leaks",
+ "description": "`freeing` is space still being reclaimed in the background after a delete. `leaked` should always be zero.",
+ "id": 40,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 109
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "bytes",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_freeing_bytes{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{pool}} freeing"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_leaked_bytes{pool=~\"$pool\"}",
+ "refId": "B",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{pool}} leaked"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_checkpoint_bytes{pool=~\"$pool\"}",
+ "refId": "C",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{pool}} checkpoint"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Dedup Ratio",
+ "description": "1.00 unless deduplication is enabled, which it usually should not be.",
+ "id": 41,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 109
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_dedup_ratio{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{pool}}"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Pool Health History",
+ "description": "Health code over time \u2014 useful for spotting a pool that briefly degraded and recovered. zfs_pool_scan_state is also collected if you want to plot scrub/resilver phases alongside it.",
+ "id": 42,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 109
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "stepAfter",
+ "lineWidth": 2,
+ "fillOpacity": 0,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "ONLINE",
+ "color": "green",
+ "index": 0
+ },
+ "1": {
+ "text": "DEGRADED",
+ "color": "orange",
+ "index": 1
+ },
+ "2": {
+ "text": "FAULTED",
+ "color": "red",
+ "index": 2
+ },
+ "3": {
+ "text": "OFFLINE",
+ "color": "text",
+ "index": 3
+ },
+ "4": {
+ "text": "UNAVAIL",
+ "color": "red",
+ "index": 4
+ },
+ "5": {
+ "text": "REMOVED",
+ "color": "purple",
+ "index": 5
+ },
+ "6": {
+ "text": "SUSPENDED",
+ "color": "red",
+ "index": 6
+ },
+ "7": {
+ "text": "AVAIL",
+ "color": "blue",
+ "index": 7
+ },
+ "8": {
+ "text": "UNKNOWN",
+ "color": "text",
+ "index": 8
+ }
+ }
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0,
+ "max": 8
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_pool_health_code{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{pool}}"
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "title": "Dataset Properties",
+ "description": "A quota of 0 means unlimited. Auto-snapshot reflects the com.sun:auto-snapshot property that zfs-auto-snapshot honours.",
+ "id": 43,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 116
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto"
+ },
+ "inspect": false,
+ "filterable": true
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Record Size"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Quota"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Ref Quota"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "Auto-snapshot"
+ },
+ "properties": [
+ {
+ "id": "mappings",
+ "value": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "no",
+ "color": "orange",
+ "index": 0
+ },
+ "1": {
+ "text": "yes",
+ "color": "green",
+ "index": 1
+ }
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "options": {
+ "showHeader": true,
+ "cellHeight": "sm",
+ "footer": {
+ "show": false,
+ "reducer": [
+ "sum"
+ ],
+ "countRows": false,
+ "fields": ""
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_dataset_recordsize_bytes{pool=~\"$pool\"}",
+ "refId": "A",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_dataset_quota_bytes{pool=~\"$pool\"}",
+ "refId": "B",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_dataset_refquota_bytes{pool=~\"$pool\"}",
+ "refId": "C",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_dataset_autosnapshot_enabled{pool=~\"$pool\"}",
+ "refId": "D",
+ "instant": true,
+ "range": false,
+ "format": "table"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "pattern": "dataset|Value.*"
+ }
+ }
+ },
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "dataset",
+ "mode": "outer"
+ }
+ },
+ {
+ "id": "organize",
+ "options": {
+ "excludeByName": {},
+ "indexByName": {},
+ "renameByName": {
+ "dataset": "Dataset",
+ "Value #A": "Record Size",
+ "Value #B": "Quota",
+ "Value #C": "Ref Quota",
+ "Value #D": "Auto-snapshot"
+ }
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "type": "row",
+ "title": "Collector Health",
+ "id": 48,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 109
+ },
+ "collapsed": true,
+ "panels": [
+ {
+ "type": "timeseries",
+ "title": "ZFS Collector Duration",
+ "description": "How long collectors/zfs-metrics.py takes. Listing every snapshot is the slow part; if this approaches the 1 minute timer interval, raise OnUnitActiveSec.",
+ "id": 45,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 110
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "s",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "zfs_metrics_collection_duration_seconds",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "collection"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Textfile Age",
+ "description": "Age of each .prom file. If this climbs without bound the collector has stopped writing and every ZFS metric above is stale.",
+ "id": 46,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 110
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "unit": "s",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "time() - node_textfile_mtime_seconds",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "{{file}}"
+ }
+ ]
+ },
+ {
+ "type": "timeseries",
+ "title": "Scrape Health",
+ "description": "node_textfile_scrape_error goes to 1 if a .prom file is malformed, in which case node_exporter drops it entirely.",
+ "id": 47,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 110
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 15,
+ "showPoints": "never",
+ "spanNulls": true,
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "gradientMode": "none",
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "up{job=\"node\"}",
+ "refId": "A",
+ "instant": false,
+ "range": true,
+ "legendFormat": "node exporter up"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "node_textfile_scrape_error",
+ "refId": "B",
+ "instant": false,
+ "range": true,
+ "legendFormat": "textfile parse error"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
diff --git a/lgtm.nix b/lgtm.nix
new file mode 100644
index 0000000..a4144ec
--- /dev/null
+++ b/lgtm.nix
@@ -0,0 +1,534 @@
+{ pkgs, config, lib, ... }:
+
+let
+ # The hostname the container uses to reach the host's exporters.
+ hostName = "poweredge";
+
+ # One scrape interval to rule them all. This MUST match the Grafana Prometheus
+ # datasource's `timeInterval` below, otherwise Grafana computes $__rate_interval
+ # from the wrong assumption and every rate()-based panel intermittently renders
+ # "No data". (Prometheus' own default is 1m, which is what broke the Node Exporter
+ # Full dashboard: $__rate_interval resolved to 60s, so rate(x[60s]) over 60s-spaced
+ # samples usually saw fewer than the two samples rate() requires.)
+ scrapeInterval = "15s";
+
+ # Where the host's zpool collector drops its metrics for node_exporter to pick up.
+ textfileDir = "/var/lib/prometheus-node-exporter-textfile";
+
+ ports = {
+ node = 9100;
+ smartctl = 9633;
+ loki = 3100;
+ prometheus = 9090;
+ grafana = 3000;
+ };
+
+ # Stable datasource UIDs, referenced by the dashboards below.
+ dsUid = { prometheus = "prometheus"; loki = "loki"; };
+
+ ##########################################################################
+ # Dashboard plumbing
+ ##########################################################################
+
+ # Dashboards exported from grafana.com carry an `__inputs` block and reference
+ # their datasource as "${DS_PROMETHEUS}". That placeholder is only substituted by
+ # Grafana's interactive *import* flow — file-based provisioning never runs it, so
+ # the uid stays literal and every panel fails to resolve a datasource. This filter
+ # performs the substitution at build time, using our fixed datasource UIDs.
+ dashboardFixup = pkgs.writeText "grafana-dashboard-fixup.jq" ''
+ def dsuid($pluginId):
+ if $pluginId == "prometheus" then $prom
+ elif $pluginId == "loki" then $loki
+ else "" end;
+
+ def subst($name; $uid):
+ walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end);
+
+ reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId)))
+ | del(.__inputs, .__requires)
+ # Provisioned dashboards must not carry an id, or Grafana rejects them on
+ # collision with whatever already occupies that id in its database.
+ | .id = null
+ '';
+
+ fixupDashboard = name: src: pkgs.runCommand "grafana-dashboard-${name}.json"
+ { nativeBuildInputs = [ pkgs.jq ]; }
+ ''
+ jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \
+ --arg loki ${lib.escapeShellArg dsUid.loki} \
+ -f ${dashboardFixup} < ${src} > "$out"
+ '';
+
+ # grafana.com dashboard, pinned by revision.
+ mkGrafanaComDashboard = { name, id, rev, sha256 }:
+ fixupDashboard name (pkgs.fetchurl {
+ url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download";
+ inherit sha256;
+ });
+
+ dashboards = {
+ # https://grafana.com/grafana/dashboards/1860 — Node Exporter Full
+ node-exporter-full = mkGrafanaComDashboard {
+ name = "node-exporter-full"; id = 1860; rev = 45;
+ sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY=";
+ };
+
+ # https://grafana.com/grafana/dashboards/22604 — Smartctl Exporter
+ smartctl-exporter = mkGrafanaComDashboard {
+ name = "smartctl-exporter"; id = 22604; rev = 2;
+ sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88=";
+ };
+
+ # Dashboard 15362 ("ZFS Pool Metrics") used to live here. It is an *InfluxDB*
+ # dashboard — its queries are InfluxQL against a `zpool_stats` measurement — so
+ # it can never render against Prometheus, no matter how it is provisioned. This
+ # is a Prometheus-native replacement built on the metrics we actually export:
+ # node_exporter's zfs collector plus the zpool textfile collector below.
+ zfs-pools = fixupDashboard "zfs-pools" ./dashboards/zfs-pools.json;
+
+ # CrystalDiskInfo-style per-drive SMART view. Generated by
+ # ./dashboards/gen_smart.py; the Good/Caution/Bad verdict it displays comes
+ # from the recording rules in ./rules/smartctl.yml.
+ smart = fixupDashboard "smart" ./dashboards/smart.json;
+ };
+
+ ##########################################################################
+ # ZFS metrics (host)
+ ##########################################################################
+
+ # node_exporter's zfs collector reads /proc/spl/kstat/zfs, which gives us ARC
+ # statistics, per-dataset objset counters and pool state -- but nothing about
+ # capacity, fragmentation, vdev errors, scrub history, compression, encryption or
+ # snapshots. OpenZFS 2.2+ also no longer publishes the per-pool `io` kstat, so
+ # node_zfs_zpool_* is largely empty. Everything else comes from the zpool/zfs CLIs
+ # via ./collectors/zfs-metrics.py, scraped through the textfile collector.
+ #
+ # The wrapper exists only to put the matching zfs userland on PATH; the collector
+ # itself is a plain script so it can be run and tested by hand.
+ snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ];
+
+ retentionArgs = lib.optionals config.services.zfs.autoSnapshot.enable (
+ lib.concatMap (label: [
+ "--retention"
+ "${label}=${toString config.services.zfs.autoSnapshot.${label}}"
+ ]) snapshotLabels
+ );
+
+ zfsCollector = pkgs.writeShellApplication {
+ name = "zfs-textfile-collector";
+ runtimeInputs = [ config.boot.zfs.package ];
+ text = ''
+ exec ${pkgs.python3}/bin/python3 ${./collectors/zfs-metrics.py} \
+ --output ${textfileDir}/zfs.prom ${lib.escapeShellArgs retentionArgs}
+ '';
+ };
+
+ # smartctl_exporter reads only the entry *count* out of each drive's self-test log
+ # and discards the log table, so it cannot say when a drive last ran a short or
+ # extended test, or whether it passed. This parses `smartctl -j -l selftest`
+ # directly. Device labels are bare names ("sda") so they join against the
+ # exporter's own series.
+ smartSelftestCollector = pkgs.writeShellApplication {
+ name = "smart-selftest-collector";
+ runtimeInputs = [ pkgs.smartmontools ];
+ text = ''
+ exec ${pkgs.python3}/bin/python3 ${./collectors/smart-selftest.py} \
+ --output ${textfileDir}/smart-selftest.prom
+ '';
+ };
+in
+{
+ ##########################################################################
+ # Host: log shipping
+ ##########################################################################
+
+ services.alloy.enable = true;
+
+ environment.etc."alloy/config.alloy".text = ''
+ loki.source.journal "journal" {
+ forward_to = [loki.write.default.receiver]
+ relabel_rules = loki.relabel.journal.rules
+ max_age = "12h"
+ }
+
+ loki.relabel "journal" {
+ forward_to = []
+ rule {
+ source_labels = ["__journal__systemd_unit"]
+ target_label = "unit"
+ }
+ rule {
+ source_labels = ["__journal__hostname"]
+ target_label = "host"
+ }
+ rule {
+ source_labels = ["__journal_priority_keyword"]
+ target_label = "level"
+ }
+ }
+
+ loki.write "default" {
+ endpoint {
+ url = "http://lgtm:${toString ports.loki}/loki/api/v1/push"
+ }
+ }
+ '';
+
+ ##########################################################################
+ # Host: telemetry sources
+ #
+ # These have to live on the host, not in the container: they read host
+ # hardware and kernel state (/proc, /sys, SMART, ZFS ioctls) that a
+ # container has no view of. Only the web services (Grafana, Prometheus,
+ # Loki) run in the container.
+ ##########################################################################
+
+ services.prometheus.exporters.node = {
+ enable = true;
+ port = ports.node;
+ listenAddress = "0.0.0.0";
+
+ # NOTE: enabledCollectors *adds* to node_exporter's defaults — cpu, diskstats,
+ # filesystem, meminfo, zfs, hwmon, pressure and friends are already on, so
+ # listing them here did nothing. Only genuinely opt-in collectors belong here.
+ enabledCollectors = [
+ "systemd" # node_systemd_* — the "Systemd" row
+ "processes" # node_processes_* — "Processes Detailed States", "PIDs", "Threads"
+ # "interrupts" # fills the "IRQ Detail" panel, but emits one series per
+ # # (cpu, irq) pair — thousands of series on a many-core box.
+ # # Enable only if you want that one panel badly enough.
+ ];
+
+ extraFlags = [
+ "--collector.textfile.directory=${textfileDir}"
+ # node_systemd_unit_start_time_seconds, so the dashboards can show when the
+ # zfs-snapshot-*/zfs-scrub units last ran rather than only whether they failed.
+ "--collector.systemd.enable-start-time-metrics"
+ ];
+ };
+
+ # Upstream hardens the unit with ProtectHome=true, which makes /home look empty and
+ # blanks out the filesystem panels for any dataset mounted under it. read-only still
+ # allows statfs while keeping the sandbox.
+ systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only";
+
+ services.prometheus.exporters.smartctl = {
+ enable = true;
+ port = ports.smartctl;
+ listenAddress = "0.0.0.0";
+ # Disks are direct-attached (AHCI/HBA), so auto-discovery via `smartctl --scan`
+ # finds them; no explicit `devices` list needed. Throttle how often we actually
+ # shell out to smartctl — SMART data changes slowly and polling spins up disks.
+ maxInterval = "60s";
+ };
+
+ # SMART self-test history. Reading the log does not spin an idle disk up, but the
+ # data only changes when a test finishes, so this runs far less often than the
+ # exporter scrapes.
+ systemd.services.smart-selftest-collector = {
+ description = "Collect SMART self-test history for the Prometheus node exporter";
+ serviceConfig = {
+ Type = "oneshot";
+ ExecStart = lib.getExe smartSelftestCollector;
+ User = "root";
+ StateDirectory = "prometheus-node-exporter-textfile";
+ StateDirectoryMode = "0755";
+ UMask = "0022";
+ TimeoutStartSec = "180s";
+
+ # Same shape as the smartctl exporter's own hardening: it needs raw device
+ # access to talk to the disks, and nothing else.
+ AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
+ CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
+ DevicePolicy = "closed";
+ DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ];
+ LockPersonality = true;
+ MemoryDenyWriteExecute = true;
+ NoNewPrivileges = true;
+ PrivateNetwork = true;
+ PrivateTmp = true;
+ ProtectControlGroups = true;
+ ProtectHome = true;
+ ProtectHostname = true;
+ ProtectKernelLogs = true;
+ ProtectKernelModules = true;
+ ProtectSystem = "strict";
+ RestrictAddressFamilies = [ "AF_UNIX" ];
+ RestrictNamespaces = true;
+ RestrictRealtime = true;
+ RestrictSUIDSGID = true;
+ SystemCallArchitectures = "native";
+ };
+ };
+
+ systemd.timers.smart-selftest-collector = {
+ description = "Collect SMART self-test history periodically";
+ wantedBy = [ "timers.target" ];
+ timerConfig = {
+ OnBootSec = "3min";
+ OnUnitActiveSec = "15min";
+ AccuracySec = "1min";
+ Unit = "smart-selftest-collector.service";
+ };
+ };
+
+ # Pool capacity and health, vdev errors, scrub history, dataset properties and
+ # snapshot retention. See zfsCollector above for why node_exporter's built-in zfs
+ # collector isn't enough on its own.
+ systemd.services.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled {
+ description = "Collect ZFS metrics for the Prometheus node exporter";
+ serviceConfig = {
+ Type = "oneshot";
+ ExecStart = lib.getExe zfsCollector;
+ User = "root";
+ StateDirectory = "prometheus-node-exporter-textfile";
+ StateDirectoryMode = "0755";
+ UMask = "0022"; # node_exporter runs as a different user and must read these
+ # Listing every snapshot on a busy pool is the slow part; keep this well
+ # above the subprocess timeout inside the collector.
+ TimeoutStartSec = "180s";
+
+ # Hardening — this only needs to read ZFS state and write one file.
+ # CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated
+ # on it; everything else is dropped.
+ CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ];
+ LockPersonality = true;
+ MemoryDenyWriteExecute = true;
+ NoNewPrivileges = true;
+ PrivateNetwork = true;
+ PrivateTmp = true;
+ ProtectControlGroups = true;
+ ProtectHome = true;
+ ProtectHostname = true;
+ ProtectKernelLogs = true;
+ ProtectKernelModules = true;
+ ProtectSystem = "strict";
+ RestrictAddressFamilies = [ "AF_UNIX" ];
+ RestrictNamespaces = true;
+ RestrictRealtime = true;
+ RestrictSUIDSGID = true;
+ SystemCallArchitectures = "native";
+ SystemCallFilter = [ "@system-service" ];
+ };
+ };
+
+ systemd.timers.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled {
+ description = "Collect ZFS metrics periodically";
+ wantedBy = [ "timers.target" ];
+ timerConfig = {
+ OnBootSec = "1min";
+ OnUnitActiveSec = "1min";
+ AccuracySec = "5s";
+ Unit = "zfs-textfile-collector.service";
+ };
+ };
+
+ # The collector used to write zpool.prom. Nothing removes a stale textfile once its
+ # writer is gone, and node_exporter would happily keep serving those frozen metrics
+ # forever, so delete it on activation.
+ systemd.tmpfiles.rules = lib.mkIf config.boot.zfs.enabled [
+ "r ${textfileDir}/zpool.prom"
+ ];
+
+ # Reachable from the container, which sits on br-lan0 with its own address.
+ # (Port 9273 is gone — that was telegraf/zpool_influxdb, replaced by the textfile
+ # collector above, which needs no listener of its own.)
+ networking.firewall.allowedTCPPorts = [ ports.node ports.smartctl ];
+
+ ##########################################################################
+ # Secrets
+ ##########################################################################
+
+ sops.secrets.grafana-secret-key = {
+ sopsFile = ./resources/secrets/grafana.yaml;
+ key = "secret_key";
+ };
+
+ ##########################################################################
+ # Container: Grafana / Prometheus / Loki
+ ##########################################################################
+
+ containers.grafana = {
+ autoStart = true;
+ privateNetwork = true;
+ hostBridge = "br-lan0";
+ localMacAddress = "02:00:00:00:00:06";
+
+ bindMounts."/run/secrets/grafana-secret_key" = {
+ hostPath = config.sops.secrets.grafana-secret-key.path;
+ isReadOnly = true;
+ };
+
+ config = { lib, pkgs, config, ... }: {
+ # Network
+ networking.enableIPv6 = false;
+ networking.interfaces.eth0.useDHCP = true;
+ networking.firewall.allowedTCPPorts = [
+ 80 # Caddy -> Grafana
+ ports.loki # host's Alloy pushes here
+ ports.prometheus # handy for debugging scrape health directly
+ ];
+
+ # ---------------------------------------------------------------
+ # Loki
+ # ---------------------------------------------------------------
+ services.loki = {
+ enable = true;
+ configuration = {
+ server = {
+ http_listen_port = ports.loki;
+ # Explicit, since the host's Alloy needs to reach it to push logs.
+ http_listen_address = "0.0.0.0";
+ };
+ auth_enabled = false;
+
+ common = {
+ path_prefix = "/var/lib/loki";
+ storage.filesystem = {
+ chunks_directory = "/var/lib/loki/chunks";
+ rules_directory = "/var/lib/loki/rules";
+ };
+ ring = {
+ instance_addr = "127.0.0.1";
+ kvstore.store = "inmemory";
+ };
+ replication_factor = 1;
+ };
+
+ schema_config.configs = [{
+ from = "2024-01-01";
+ store = "tsdb";
+ object_store = "filesystem";
+ schema = "v13";
+ index = { prefix = "index_"; period = "24h"; };
+ }];
+ };
+ };
+
+ # ---------------------------------------------------------------
+ # Prometheus
+ # ---------------------------------------------------------------
+ services.prometheus = {
+ enable = true;
+
+ # Without this Prometheus falls back to its built-in 1m scrape interval,
+ # which desynchronises from what Grafana assumes and breaks every
+ # rate()-based panel. See the comment on `scrapeInterval` at the top.
+ globalConfig = {
+ scrape_interval = scrapeInterval;
+ evaluation_interval = scrapeInterval;
+ scrape_timeout = "10s";
+ };
+
+ # Drive health verdict. Kept as recording rules rather than inlined in
+ # dashboard panels so the logic is written once and reviewable on its
+ # own. services.prometheus.checkConfig runs promtool at build time, so
+ # a broken rule fails the build instead of the service.
+ ruleFiles = [ ./rules/smartctl.yml ];
+
+ scrapeConfigs = [
+ {
+ job_name = "node";
+ static_configs = [{ targets = [ "${hostName}:${toString ports.node}" ]; }];
+ }
+ {
+ job_name = "smartctl";
+ # Querying SMART is comparatively slow and the data barely moves, so
+ # this job opts out of the global 15s. Safe to do here because the
+ # smartctl dashboard uses no rate() queries at all.
+ scrape_interval = "60s";
+ scrape_timeout = "30s";
+ static_configs = [{ targets = [ "${hostName}:${toString ports.smartctl}" ]; }];
+ }
+ ];
+ };
+
+ # ---------------------------------------------------------------
+ # Grafana
+ # https://wiki.nixos.org/wiki/Grafana
+ # ---------------------------------------------------------------
+ services.grafana = {
+ enable = true;
+ settings.server = { http_addr = "0.0.0.0"; http_port = ports.grafana; };
+
+ provision = {
+ enable = true;
+
+ # Mutable dashboard provider pulling from /etc/grafana-dashboards, so
+ # dashboards can be declared with environment.etc below.
+ dashboards.settings.providers = [{
+ name = "my dashboards";
+ disableDeletion = true;
+ options = {
+ path = "/etc/grafana-dashboards";
+ foldersFromFilesStructure = true;
+ };
+ }];
+
+ # Grafana cannot change the UID of an existing datasource in place. Its
+ # provisioner finds the datasource by name, then calls UpdateDataSource,
+ # which re-fetches using `id AND uid` — with a new uid that row doesn't
+ # match, so it fails with "data source not found" and Grafana refuses to
+ # start. Deleting first turns the upsert into a plain insert; the
+ # provisioner runs deletes before inserts for exactly this case, and
+ # suppresses the deletion event because the datasource is about to be
+ # re-created. Safe to leave in permanently: both datasources are fully
+ # declarative, and dashboards bind to them by UID, not by database ID.
+ datasources.settings.deleteDatasources = [
+ { name = "Prometheus"; orgId = 1; }
+ { name = "Loki"; orgId = 1; }
+ ];
+
+ datasources.settings.datasources = [
+ {
+ name = "Prometheus";
+ type = "prometheus";
+ uid = dsUid.prometheus; # fixed, so dashboards can reference it
+ url = "http://127.0.0.1:${toString ports.prometheus}";
+ isDefault = true;
+ jsonData = {
+ # Tells Grafana how often the data is actually scraped. This is
+ # what $__rate_interval is derived from; leaving it unset makes
+ # Grafana guess 15s and mis-size every rate() window.
+ timeInterval = scrapeInterval;
+ httpMethod = "POST";
+ };
+ }
+ {
+ name = "Loki";
+ type = "loki";
+ uid = dsUid.loki;
+ url = "http://127.0.0.1:${toString ports.loki}";
+ }
+ ];
+ };
+ };
+
+ # Grafana secret key
+ systemd.services.grafana.serviceConfig.LoadCredential = [
+ "secret_key:/run/secrets/grafana-secret_key"
+ ];
+ services.grafana.settings.security.secret_key =
+ "$__file{/run/credentials/grafana.service/secret_key}";
+
+ # ---------------------------------------------------------------
+ # Dashboards
+ # ---------------------------------------------------------------
+ environment.etc = lib.mapAttrs' (name: drv:
+ lib.nameValuePair "grafana-dashboards/${name}.json" { source = drv; }
+ ) dashboards;
+
+ # ---------------------------------------------------------------
+ # Reverse proxy
+ # ---------------------------------------------------------------
+ services.caddy = {
+ enable = true;
+ virtualHosts.":80".extraConfig = ''
+ reverse_proxy localhost:${toString ports.grafana}
+ '';
+ };
+
+ system.stateVersion = "26.05";
+ };
+ };
+}
diff --git a/rules/smartctl.yml b/rules/smartctl.yml
new file mode 100644
index 0000000..dafed8d
--- /dev/null
+++ b/rules/smartctl.yml
@@ -0,0 +1,116 @@
+# Drive health verdict, in the spirit of CrystalDiskInfo's Good / Caution / Bad.
+#
+# This lives in recording rules rather than in dashboard panels so the logic is written
+# once and is reviewable on its own. The panels then just read smartctl:health_code.
+#
+# The fleet is mixed: SATA drives expose the classic ATA attribute table, SAS drives
+# expose none of it and report health through SCSI error counters instead. Every rule
+# below is therefore made *total* -- it ends in `or (count by (device) (...) * 0)` so
+# that each series exists with value 0 for every drive, including drives of the other
+# kind. Without that, the arithmetic in smartctl:health_code would silently drop drives
+# whose label sets did not match on both sides.
+
+groups:
+ - name: smartctl
+ interval: 1m
+ rules:
+ # --- ATA ------------------------------------------------------------------
+ # An attribute is failing when its normalised current value has fallen to or
+ # below the manufacturer's threshold. Thresholds of 0 mean "informational, never
+ # fails", so those are excluded rather than treated as an instant failure.
+ - record: smartctl:attr_failing:count
+ expr: |
+ count by (device) (
+ smartctl_device_attribute{attribute_value_type="value"}
+ <= on(device, attribute_id)
+ (smartctl_device_attribute{attribute_value_type="thresh"} > 0)
+ )
+ or (count by (device) (smartctl_device_smart_status) * 0)
+
+ # Attributes whose raw value is a genuine warning sign across vendors:
+ # 5 Reallocated_Sector_Ct 187 Reported_Uncorrect
+ # 188 Command_Timeout 197 Current_Pending_Sector
+ # 198 Offline_Uncorrectable 199 UDMA_CRC_Error_Count (cabling)
+ # Deliberately narrow: attributes like 1 (Raw_Read_Error_Rate) carry huge raw
+ # values on Seagate drives by design and would flag healthy disks.
+ - record: smartctl:attr_caution:count
+ expr: |
+ count by (device) (
+ smartctl_device_attribute{
+ attribute_value_type="raw",
+ attribute_id=~"5|187|188|197|198|199"
+ } > 0
+ )
+ or (count by (device) (smartctl_device_smart_status) * 0)
+
+ # --- SAS / SCSI -----------------------------------------------------------
+ # Grown defect list: sectors reallocated since the drive left the factory. The
+ # SAS equivalent of attribute 5. Non-zero is worth watching, growth is worth
+ # acting on.
+ - record: smartctl:scsi_defects:count
+ expr: |
+ sum by (device) (smartctl_scsi_grown_defect_list)
+ or (count by (device) (smartctl_device_smart_status) * 0)
+
+ # Errors the drive could not correct at all, which means data was lost.
+ - record: smartctl:scsi_uncorrected:count
+ expr: |
+ sum by (device) (smartctl_read_total_uncorrected_errors)
+ + sum by (device) (smartctl_write_total_uncorrected_errors)
+ or (count by (device) (smartctl_device_smart_status) * 0)
+
+ # --- Verdict --------------------------------------------------------------
+ # 0 good, 1 caution, 2 bad. The inner clamp_max on each group keeps two
+ # simultaneous caution signals from adding up to a false "bad".
+ - record: smartctl:health_code
+ expr: |
+ clamp_max(
+ 2 * clamp_max(
+ (max by (device) (smartctl_device_smart_status) == bool 0)
+ + (smartctl:attr_failing:count > bool 0)
+ + (smartctl:scsi_uncorrected:count > bool 0)
+ , 1)
+ + clamp_max(
+ (smartctl:attr_caution:count > bool 0)
+ + (smartctl:scsi_defects:count > bool 0)
+ , 1)
+ , 2)
+
+ # --- Row labels -----------------------------------------------------------
+ # Which section of the SMART dashboard a drive belongs in. Keyed on metric
+ # presence rather than the `protocol` label: only ATA drives report an attribute
+ # table, and only SCSI drives report a grown defect list.
+ - record: smartctl:bus
+ expr: |
+ label_replace(count by (device) (smartctl_device_attribute), "bus", "ata", "", "")
+ or
+ label_replace(count by (device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "")
+
+ # The identity string from collectors/smart-selftest.py, made total so a drive
+ # still gets a row if that collector has not written its file yet. `unless` (not
+ # `or`) is what keeps the two halves disjoint -- both sides carry a `label`, so
+ # `or` would emit two series per drive rather than deduplicating.
+ - record: smartctl:ident
+ expr: |
+ max by (device, label) (smart_drive_label)
+ or
+ label_replace(
+ count by (device) (smartctl_device_smart_status)
+ unless on(device) smart_drive_label,
+ "label", "identity pending", "", "")
+
+ # One preformatted line per drive: "sde · GOOD · 6.0 TB · SAS 7.2K · model · SN".
+ # Grafana can interpolate a variable's *display text* into a repeated row title
+ # (${sas_device:text}), which is the only way to label a collapsed row with live
+ # data -- so the whole line has to exist as a single label here.
+ - record: smartctl:row
+ expr: |
+ label_join(
+ (
+ label_replace(smartctl:health_code == 0, "verdict", "GOOD", "", "")
+ or label_replace(smartctl:health_code == 1, "verdict", "CAUTION", "", "")
+ or label_replace(smartctl:health_code == 2, "verdict", "BAD", "", "")
+ )
+ * on(device) group_left(bus) smartctl:bus
+ * on(device) group_left(label) smartctl:ident,
+ "row", " · ", "device", "verdict", "label")