initial commit

1 files changed, 617 insertions, 0 deletions

diff --git a/dashboards/gen_zfs.py b/dashboards/gen_zfs.py
new file mode 100644
index 0000000..4edb38b
--- /dev/null
+++ b/dashboards/gen_zfs.py
@@ -0,0 +1,617 @@
+#!/usr/bin/env python3
+"""Generate the "ZFS Pools" Grafana dashboard.
+
+Run this and commit the result:
+
+    python3 dashboards/gen_zfs.py > dashboards/zfs-pools.json
+
+Hand-writing ~2000 lines of Grafana panel JSON invites subtle inconsistencies, so
+panels are built from a handful of helpers instead -- every timeseries goes through one
+code path and is therefore structurally identical by construction.
+
+The queries here are deliberately limited to metrics we actually produce:
+
+* zfs_pool_* / zfs_dataset_* / zfs_snapshot_*  -- collectors/zfs-metrics.py
+* node_zfs_arc_* / node_zfs_zpool_dataset_*    -- node_exporter's zfs collector
+* node_systemd_*                               -- node_exporter's systemd collector
+
+If you add a panel, add the metric to the collector first and keep this list honest.
+"""
+
+import json
+
+from panels import (DS, OK_BAD, bargauge, gauge, gp, row, stat, table, timeseries)
+
+
+STATE_MAP = [{
+    "type": "value",
+    "options": {
+        "0": {"text": "ONLINE", "color": "green", "index": 0},
+        "1": {"text": "DEGRADED", "color": "orange", "index": 1},
+        "2": {"text": "FAULTED", "color": "red", "index": 2},
+        "3": {"text": "OFFLINE", "color": "text", "index": 3},
+        "4": {"text": "UNAVAIL", "color": "red", "index": 4},
+        "5": {"text": "REMOVED", "color": "purple", "index": 5},
+        "6": {"text": "SUSPENDED", "color": "red", "index": 6},
+        "7": {"text": "AVAIL", "color": "blue", "index": 7},
+        "8": {"text": "UNKNOWN", "color": "text", "index": 8},
+    },
+}]
+
+BOOL_MAP = [{
+    "type": "value",
+    "options": {
+        "0": {"text": "idle", "color": "green", "index": 0},
+        "1": {"text": "RUNNING", "color": "blue", "index": 1},
+    },
+}]
+
+POOL = 'pool=~"$pool"'
+panels = []
+
+# ======================================================================================
+# Pool Overview
+# ======================================================================================
+panels.append(row("Pool Overview", 0))
+
+panels.append(stat(
+    "Pool Health", [(f'zfs_pool_health_code{{{POOL}}}', "{{pool}}")],
+    gp(0, 1, 4, 4), mappings=STATE_MAP, colormode="background",
+    desc="Health as reported by `zpool list -o health`.",
+))
+panels.append(stat(
+    "Data Errors", [(f'sum(zfs_pool_data_errors{{{POOL}}}) or vector(0)', "errors")],
+    gp(4, 1, 4, 4), thresholds=OK_BAD, colormode="background",
+    desc="Permanent, unrepairable errors from `zpool status`. Anything above zero "
+         "means data has been lost and the affected files need restoring.",
+))
+panels.append(stat(
+    "vdev Errors",
+    [(f'sum(zfs_pool_vdev_errors_total{{{POOL}}}) or vector(0)', "errors")],
+    gp(8, 1, 4, 4), thresholds=OK_BAD, colormode="background",
+    desc="Total read + write + checksum errors across all vdevs. Cleared by "
+         "`zpool clear`.",
+))
+panels.append(stat(
+    "Last Scrub",
+    [(f'time() - max(zfs_pool_last_scrub_end_timestamp_seconds{{{POOL}}})', "ago")],
+    gp(12, 1, 4, 4), unit="s", colormode="background",
+    thresholds={"mode": "absolute", "steps": [
+        {"color": "green", "value": None},
+        {"color": "yellow", "value": 35 * 86400},
+        {"color": "red", "value": 70 * 86400},
+    ]},
+    desc="Time since the most recent completed scrub. Turns amber past ~5 weeks, "
+         "which is a monthly scrub schedule having missed a run.",
+))
+panels.append(stat(
+    "Scrub / Resilver", [
+        (f'max(zfs_pool_scrub_in_progress{{{POOL}}})', "scrub"),
+        (f'max(zfs_pool_resilver_in_progress{{{POOL}}})', "resilver"),
+    ],
+    gp(16, 1, 4, 4), mappings=BOOL_MAP, colormode="background",
+))
+panels.append(stat(
+    "Locked Datasets",
+    [(f'count(zfs_dataset_encrypted{{{POOL}}} == 1 and '
+      f'zfs_dataset_key_available{{{POOL}}} == 0) or vector(0)', "locked")],
+    gp(20, 1, 4, 4), thresholds=OK_BAD, colormode="background",
+    desc="Encrypted datasets whose key is not loaded. These are unreadable and are "
+         "silently skipped by anything that walks the filesystem, including backups.",
+))
+
+panels.append(gauge(
+    "Space Used", [(f'zfs_pool_capacity_ratio{{{POOL}}}', "{{pool}}")],
+    gp(0, 5, 5, 4),
+    desc="How full the pool is: allocated / total. ZFS write performance degrades "
+         "sharply past ~80%, and the pool becomes effectively read-only near 100%. "
+         "The byte figures behind this are in 'Capacity by Pool' below.",
+))
+panels.append(gauge(
+    "Fragmentation", [(f'zfs_pool_fragmentation_ratio{{{POOL}}}', "{{pool}}")],
+    gp(5, 5, 5, 4),
+    thresholds={"mode": "absolute", "steps": [
+        {"color": "green", "value": None},
+        {"color": "yellow", "value": 0.5},
+        {"color": "red", "value": 0.75},
+    ]},
+    desc="Free-space fragmentation.",
+))
+panels.append(stat(
+    "Free Space", [(f'zfs_pool_free_bytes{{{POOL}}}', "{{pool}}")],
+    gp(10, 5, 5, 4), unit="bytes",
+))
+panels.append(stat(
+    "Snapshots Overdue",
+    [(f'count((time() - zfs_snapshot_newest_timestamp_seconds{{{POOL}}}) > on(label) '
+      f'group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)', "overdue")],
+    gp(15, 5, 4, 4), thresholds=OK_BAD, colormode="background",
+    desc="Dataset/label pairs whose newest snapshot is more than two intervals old — "
+         "i.e. the auto-snapshot timer has missed at least one run.",
+))
+panels.append(stat(
+    "Retention Depth",
+    [(f'time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{{{POOL}}})', "oldest")],
+    gp(19, 5, 5, 4), unit="s",
+    desc="Age of the oldest snapshot on the pool — how far back you can actually "
+         "restore from.",
+))
+
+panels.append(timeseries(
+    "Pool Space Used / Free", [
+        (f'zfs_pool_allocated_bytes{{{POOL}}}', "{{pool}} used"),
+        (f'zfs_pool_free_bytes{{{POOL}}}', "{{pool}} free"),
+    ],
+    gp(0, 9, 12, 8), unit="bytes", stack=True, minval=0,
+    desc="Stacked, so the top of each pool's band is its total size and the split "
+         "between the bands is the usage fraction the gauge above shows.",
+))
+panels.append(timeseries(
+    "Fragmentation",
+    [(f'zfs_pool_fragmentation_ratio{{{POOL}}}', "{{pool}}")],
+    gp(12, 9, 12, 8), unit="percentunit", minval=0, maxval=1,
+    desc="Free-space fragmentation — how scattered the pool's remaining free space "
+         "is, not how scattered the data is. It rises as a pool fills and stays high "
+         "afterwards; only rewriting data (send/recv, or deleting a lot) brings it "
+         "down. Kept on its own axis because it is unrelated to space used, and "
+         "plotting the two together invites reading one as the other.",
+))
+
+panels.append(table(
+    "Capacity by Pool", [
+        (f'zfs_pool_size_bytes{{{POOL}}}', None),
+        (f'max by (pool) (zfs_pool_allocated_bytes{{{POOL}}})', None),
+        (f'max by (pool) (zfs_pool_free_bytes{{{POOL}}})', None),
+        (f'max by (pool) (zfs_pool_capacity_ratio{{{POOL}}})', None),
+        (f'max by (pool) (zfs_pool_fragmentation_ratio{{{POOL}}})', None),
+        (f'max by (pool) (zfs_pool_snapshot_used_bytes{{{POOL}}})', None),
+        (f'max by (pool) (zfs_pool_dedup_ratio{{{POOL}}})', None),
+    ],
+    gp(0, 17, 24, 8), keep="pool|Value.*",
+    renames={
+        "__join__": "pool", "pool": "Pool",
+        "Value #A": "Size", "Value #B": "Used", "Value #C": "Free",
+        "Value #D": "Used %", "Value #E": "Fragmentation",
+        "Value #F": "Snapshots", "Value #G": "Dedup",
+    },
+    order=["pool", "Value #A", "Value #B", "Value #C", "Value #D", "Value #E",
+           "Value #F", "Value #G"],
+    sort_by="Used %",
+    overrides=[
+        # Anchored: byRegexp is a substring match, so a bare "Used" would also
+        # capture "Used %" and fight the percent override below.
+        {"matcher": {"id": "byRegexp", "options": "^(Size|Used|Free|Snapshots)$"},
+         "properties": [{"id": "unit", "value": "bytes"}]},
+        {"matcher": {"id": "byName", "options": "Used %"},
+         "properties": [
+             {"id": "unit", "value": "percentunit"},
+             {"id": "max", "value": 1},
+             {"id": "thresholds", "value": {"mode": "absolute", "steps": [
+                 {"color": "green", "value": None},
+                 {"color": "yellow", "value": 0.8},
+                 {"color": "red", "value": 0.9},
+             ]}},
+             {"id": "custom.cellOptions",
+              "value": {"type": "gauge", "mode": "gradient"}},
+         ]},
+        {"matcher": {"id": "byName", "options": "Fragmentation"},
+         "properties": [{"id": "unit", "value": "percentunit"}]},
+    ],
+    desc="Per-pool space in bytes. 'Size' is the raw pool size zpool reports, which "
+         "for raidz is before parity overhead — so 'Free' is the number to trust for "
+         "how much you can still write. 'Snapshots' is space held only by snapshots.",
+))
+
+# ======================================================================================
+# Integrity & Errors
+# ======================================================================================
+panels.append(row("Integrity & Errors", 25))
+
+panels.append(table(
+    "vdev State", [(f'zfs_pool_vdev_state_code{{{POOL}}}', None)],
+    gp(0, 26, 12, 9), keep="pool|vdev|Value",
+    renames={"pool": "Pool", "vdev": "vdev", "Value": "State"},
+    mappings=STATE_MAP,
+    overrides=[{
+        "matcher": {"id": "byName", "options": "State"},
+        "properties": [{"id": "custom.cellOptions",
+                        "value": {"type": "color-background", "mode": "basic"}}],
+    }],
+    desc="Every vdev and leaf device in the selected pool(s).",
+))
+panels.append(timeseries(
+    "vdev Error Counters",
+    [(f'zfs_pool_vdev_errors_total{{{POOL}}}', "{{pool}}/{{vdev}} {{type}}")],
+    gp(12, 26, 12, 9), minval=0,
+    custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+    desc="Cumulative per-vdev errors. A counter that climbs is a disk on its way out.",
+))
+
+panels.append(table(
+    "Scrub & Resilver History", [
+        (f'zfs_pool_last_scrub_end_timestamp_seconds{{{POOL}}} * 1000', None),
+        (f'zfs_pool_last_scrub_duration_seconds{{{POOL}}}', None),
+        (f'zfs_pool_last_scrub_errors{{{POOL}}}', None),
+        (f'zfs_pool_last_scrub_repaired_bytes{{{POOL}}}', None),
+        # zpool status reports one scan per pool, so a pool whose most recent scan was
+        # a resilver has no scrub record at all and would otherwise vanish from this
+        # table. The outer join keeps it, with only this column filled.
+        (f'zfs_pool_last_resilver_end_timestamp_seconds{{{POOL}}} * 1000', None),
+    ],
+    gp(0, 35, 12, 7), keep="pool|Value.*",
+    renames={
+        "__join__": "pool",
+        "pool": "Pool",
+        "Value #A": "Scrub Finished", "Value #B": "Duration",
+        "Value #C": "Errors", "Value #D": "Repaired",
+        "Value #E": "Resilver Finished",
+    },
+    overrides=[
+        {"matcher": {"id": "byName", "options": "Scrub Finished"},
+         "properties": [{"id": "unit", "value": "dateTimeAsIso"}]},
+        {"matcher": {"id": "byName", "options": "Resilver Finished"},
+         "properties": [{"id": "unit", "value": "dateTimeAsIso"}]},
+        {"matcher": {"id": "byName", "options": "Duration"},
+         "properties": [{"id": "unit", "value": "s"}]},
+        {"matcher": {"id": "byName", "options": "Repaired"},
+         "properties": [{"id": "unit", "value": "bytes"}]},
+        {"matcher": {"id": "byName", "options": "Errors"},
+         "properties": [
+             {"id": "custom.cellOptions",
+              "value": {"type": "color-background", "mode": "basic"}},
+             {"id": "thresholds", "value": OK_BAD},
+         ]},
+    ],
+    desc="Result of the last completed scrub per pool. Errors here are blocks that "
+         "could not be repaired from redundancy.",
+))
+panels.append(timeseries(
+    "Pool Data Errors", [(f'zfs_pool_data_errors{{{POOL}}}', "{{pool}}")],
+    gp(12, 35, 12, 7), minval=0,
+    custom={"lineWidth": 2, "fillOpacity": 10, "showPoints": "auto"},
+    desc="Permanent data errors over time.",
+))
+
+# ======================================================================================
+# Snapshots & Retention
+# ======================================================================================
+panels.append(row("Snapshots & Retention", 42))
+
+panels.append(bargauge(
+    "Time Since Last Snapshot",
+    [(f'time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{{{POOL}}})',
+      "{{label}}")],
+    gp(0, 43, 8, 8), unit="s",
+    thresholds={"mode": "absolute", "steps": [
+        {"color": "green", "value": None},
+        {"color": "yellow", "value": 2 * 3600},
+        {"color": "red", "value": 2 * 86400},
+    ]},
+    desc="Age of the newest snapshot of each label. Compare against the label's own "
+         "cadence — 'frequent' is every 15 minutes, 'monthly' every month, so the "
+         "colour thresholds here are only a coarse hint.",
+))
+panels.append(bargauge(
+    "Snapshots vs Retention Target",
+    [(f'min by (label) (zfs_snapshot_count{{{POOL}}} / on(label) group_left() '
+      f'zfs_snapshot_retention_target)', "{{label}}")],
+    gp(8, 43, 8, 8), unit="percentunit", maxv=1.2,
+    thresholds={"mode": "absolute", "steps": [
+        {"color": "red", "value": None},
+        {"color": "yellow", "value": 0.5},
+        {"color": "green", "value": 0.9},
+    ]},
+    desc="Worst dataset's snapshot count as a fraction of the configured "
+         "services.zfs.autoSnapshot retention. 100% means the full set is on disk; a "
+         "freshly-created dataset legitimately sits below target until it fills up.",
+))
+panels.append(bargauge(
+    "Oldest Snapshot",
+    [(f'time() - min by (label) '
+      f'(zfs_pool_snapshot_oldest_timestamp_seconds{{{POOL}}})', "{{label}}")],
+    gp(16, 43, 8, 8), unit="s",
+    desc="How far back each label reaches.",
+))
+
+panels.append(timeseries(
+    "Snapshot Count by Label",
+    [(f'sum by (label) (zfs_snapshot_count{{{POOL}}})', "{{label}}")],
+    gp(0, 51, 12, 8), minval=0,
+    desc="Total snapshots per label. A steady sawtooth is healthy rotation; a "
+         "monotonic decline means snapshots are expiring but not being replaced.",
+))
+panels.append(timeseries(
+    "Snapshot Space by Label",
+    [(f'sum by (label) (zfs_pool_snapshot_used_bytes{{{POOL}}})', "{{label}}")],
+    gp(12, 51, 12, 8), unit="bytes", stack=True, minval=0,
+    desc="Space held down by snapshots — the cost of your retention policy.",
+))
+
+panels.append(stat(
+    "Snapshot / Scrub Job Failures",
+    [('sum(node_systemd_unit_state{name=~"zfs-(snapshot|scrub).*",state="failed"}) '
+      'or vector(0)', "failed units")],
+    gp(0, 59, 6, 6), thresholds=OK_BAD, colormode="background",
+    desc="systemd units for the auto-snapshot and auto-scrub timers currently in a "
+         "failed state.",
+))
+panels.append(bargauge(
+    "Time Since Timer Last Fired",
+    [('time() - node_systemd_timer_last_trigger_seconds{name=~"zfs-.*"}', "{{name}}")],
+    gp(6, 59, 18, 6), unit="s",
+    desc="When each zfs-* systemd timer last triggered. This is the job side of the "
+         "picture; the snapshot ages above are the result side.",
+))
+
+# ======================================================================================
+# Datasets
+# ======================================================================================
+panels.append(row("Datasets", 65))
+
+panels.append(timeseries(
+    "Dataset Usage Over Time",
+    [(f'zfs_dataset_used_bytes{{{POOL}}}', "{{dataset}}")],
+    gp(0, 66, 24, 9), unit="bytes", minval=0,
+    custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+    desc="Space each dataset is responsible for, including its snapshots and "
+         "descendants — so a parent's line contains its children's. Unstacked and "
+         "unfilled so a single dataset's growth is readable against the others rather "
+         "than riding on top of them. The legend carries last and max per dataset; the "
+         "instant breakdown is in 'Capacity by Dataset' below.",
+))
+
+panels.append(table(
+    "Capacity by Dataset", [
+        # Only this query keeps `pool` as its own column; the rest are reduced to the
+        # join key so joinByField has nothing to collide on. `dataset` is the full
+        # path ("ingens/media"), so it is unique on its own.
+        (f'zfs_dataset_used_bytes{{{POOL}}}', None),
+        (f'max by (dataset) (zfs_dataset_used_by_dataset_bytes{{{POOL}}})', None),
+        (f'max by (dataset) (zfs_dataset_used_by_snapshots_bytes{{{POOL}}})', None),
+        (f'max by (dataset) (zfs_dataset_referenced_bytes{{{POOL}}})', None),
+        (f'max by (dataset) (zfs_dataset_logical_used_bytes{{{POOL}}})', None),
+        (f'max by (dataset) (zfs_dataset_available_bytes{{{POOL}}})', None),
+        (f'max by (dataset) (zfs_dataset_compression_ratio{{{POOL}}})', None),
+        # Quota is 0 when unset, which would read as "no space allowed"; blank it.
+        (f'max by (dataset) (zfs_dataset_quota_bytes{{{POOL}}} > 0)', None),
+    ],
+    gp(0, 75, 24, 8), keep="pool|dataset|Value.*",
+    renames={
+        "__join__": "dataset", "dataset": "Dataset", "pool": "Pool",
+        "Value #A": "Used", "Value #B": "Own Data", "Value #C": "Snapshots",
+        "Value #D": "Referenced", "Value #E": "Logical", "Value #F": "Available",
+        "Value #G": "Ratio", "Value #H": "Quota",
+    },
+    order=["pool", "dataset", "Value #A", "Value #B", "Value #C", "Value #D",
+           "Value #E", "Value #F", "Value #G", "Value #H"],
+    sort_by="Used",
+    overrides=[
+        {"matcher": {"id": "byRegexp",
+                     "options": "^(Used|Own Data|Snapshots|Referenced|Logical|"
+                                "Available|Quota)$"},
+         "properties": [{"id": "unit", "value": "bytes"}]},
+        {"matcher": {"id": "byName", "options": "Quota"},
+         "properties": [{"id": "noValue", "value": "none"}]},
+        {"matcher": {"id": "byName", "options": "Ratio"},
+         "properties": [{"id": "decimals", "value": 2}]},
+    ],
+    desc="Per-dataset space, separate from the per-pool figures above. 'Used' is the "
+         "total a dataset is responsible for; 'Own Data' and 'Snapshots' are the two "
+         "halves of that. 'Logical' is the size before compression, so Logical / "
+         "Referenced is where the compression ratio comes from. 'Available' is shared "
+         "with every sibling dataset unless a quota says otherwise.",
+))
+
+panels.append(table(
+    "Compression",
+    [(f'zfs_dataset_compression_ratio{{{POOL}}} * on(pool, dataset) '
+      f'group_left(compression) zfs_dataset_compression{{{POOL}}}', None)],
+    gp(0, 83, 12, 8), keep="pool|dataset|compression|Value",
+    order=["pool", "dataset", "compression", "Value"],
+    renames={"pool": "Pool", "dataset": "Dataset",
+             "compression": "Algorithm", "Value": "Ratio"},
+    sort_by="Ratio",
+    desc="group_left() attaches the algorithm label from the info metric onto the "
+         "ratio, so this needs only one query and no join transformation.",
+))
+panels.append(table(
+    "Encryption",
+    [(f'zfs_dataset_keystatus_code{{{POOL}}} * on(pool, dataset) '
+      f'group_left(encryption) zfs_dataset_encryption{{{POOL}}}', None)],
+    gp(12, 83, 12, 8), keep="pool|dataset|encryption|Value",
+    order=["pool", "dataset", "encryption", "Value"],
+    renames={"pool": "Pool", "dataset": "Dataset",
+             "encryption": "Suite", "Value": "Key"},
+    mappings=[{"type": "value", "options": {
+        "0": {"text": "—", "color": "text", "index": 0},
+        "1": {"text": "unlocked", "color": "green", "index": 1},
+        "2": {"text": "LOCKED", "color": "red", "index": 2},
+        "3": {"text": "unknown", "color": "orange", "index": 3},
+    }}],
+    overrides=[{
+        "matcher": {"id": "byName", "options": "Key"},
+        "properties": [{"id": "custom.cellOptions",
+                        "value": {"type": "color-background", "mode": "basic"}}],
+    }],
+    desc="Encryption suite per dataset and the state of its key. Datasets that are "
+         "not encrypted show suite 'off' and a dash — only an encrypted dataset can "
+         "be locked.",
+))
+
+# ======================================================================================
+# Performance
+# ======================================================================================
+panels.append(row("Performance", 91))
+
+panels.append(timeseries(
+    "Dataset Throughput", [
+        (f'rate(node_zfs_zpool_dataset_nread{{zpool=~"$pool"}}[$__rate_interval])',
+         "{{zpool}}/{{dataset}} read"),
+        (f'-rate(node_zfs_zpool_dataset_nwritten{{zpool=~"$pool"}}[$__rate_interval])',
+         "{{zpool}}/{{dataset}} write"),
+    ],
+    gp(0, 92, 12, 8), unit="Bps",
+    desc="Per-dataset throughput from node_exporter's objset kstats. Writes are drawn "
+         "negative.",
+))
+panels.append(timeseries(
+    "Dataset IOPS", [
+        (f'rate(node_zfs_zpool_dataset_reads{{zpool=~"$pool"}}[$__rate_interval])',
+         "{{zpool}}/{{dataset}} read"),
+        (f'-rate(node_zfs_zpool_dataset_writes{{zpool=~"$pool"}}[$__rate_interval])',
+         "{{zpool}}/{{dataset}} write"),
+    ],
+    gp(12, 92, 12, 8), unit="iops",
+    desc="Writes are drawn negative.",
+))
+
+panels.append(timeseries(
+    "ARC Size", [
+        ("node_zfs_arc_size", "size"),
+        ("node_zfs_arc_c_max", "target max (c_max)"),
+        ("node_zfs_arc_c_min", "target min (c_min)"),
+    ],
+    gp(0, 100, 8, 8), unit="bytes", minval=0,
+))
+panels.append(timeseries(
+    "ARC Hit Ratio",
+    [("rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min("
+      "rate(node_zfs_arc_hits[$__rate_interval]) + "
+      "rate(node_zfs_arc_misses[$__rate_interval]), 1)", "hit ratio")],
+    gp(8, 100, 8, 8), unit="percentunit", minval=0,
+    desc="clamp_min avoids a divide-by-zero while the pool is idle.",
+))
+panels.append(timeseries(
+    "ARC Hits / Misses", [
+        ("rate(node_zfs_arc_hits[$__rate_interval])", "hits"),
+        ("rate(node_zfs_arc_misses[$__rate_interval])", "misses"),
+    ],
+    gp(16, 100, 8, 8), unit="ops", minval=0,
+))
+
+# ======================================================================================
+# Collapsed: advanced / diagnostic
+# ======================================================================================
+panels.append(row("Pool Internals", 108, collapsed=True, children=[
+    timeseries(
+        "Reclaim & Leaks", [
+            (f'zfs_pool_freeing_bytes{{{POOL}}}', "{{pool}} freeing"),
+            (f'zfs_pool_leaked_bytes{{{POOL}}}', "{{pool}} leaked"),
+            (f'zfs_pool_checkpoint_bytes{{{POOL}}}', "{{pool}} checkpoint"),
+        ],
+        gp(0, 109, 8, 7), unit="bytes", minval=0,
+        desc="`freeing` is space still being reclaimed in the background after a "
+             "delete. `leaked` should always be zero.",
+    ),
+    timeseries(
+        "Dedup Ratio", [(f'zfs_pool_dedup_ratio{{{POOL}}}', "{{pool}}")],
+        gp(8, 109, 8, 7), minval=0,
+        desc="1.00 unless deduplication is enabled, which it usually should not be.",
+    ),
+    timeseries(
+        "Pool Health History", [
+            (f'zfs_pool_health_code{{{POOL}}}', "{{pool}}"),
+        ],
+        # Bounded to the state-code range so the y-axis reads as states, not 0-100.
+        gp(16, 109, 8, 7), mappings=STATE_MAP, minval=0, maxval=8,
+        custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"},
+        desc="Health code over time — useful for spotting a pool that briefly "
+             "degraded and recovered. zfs_pool_scan_state is also collected if you "
+             "want to plot scrub/resilver phases alongside it.",
+    ),
+    table(
+        "Dataset Properties", [
+            (f'zfs_dataset_recordsize_bytes{{{POOL}}}', None),
+            (f'zfs_dataset_quota_bytes{{{POOL}}}', None),
+            (f'zfs_dataset_refquota_bytes{{{POOL}}}', None),
+            (f'zfs_dataset_autosnapshot_enabled{{{POOL}}}', None),
+        ],
+        gp(0, 116, 24, 8), keep="dataset|Value.*",
+        renames={
+            "__join__": "dataset", "dataset": "Dataset",
+            "Value #A": "Record Size", "Value #B": "Quota",
+            "Value #C": "Ref Quota", "Value #D": "Auto-snapshot",
+        },
+        overrides=[
+            {"matcher": {"id": "byName", "options": "Record Size"},
+             "properties": [{"id": "unit", "value": "bytes"}]},
+            {"matcher": {"id": "byName", "options": "Quota"},
+             "properties": [{"id": "unit", "value": "bytes"}]},
+            {"matcher": {"id": "byName", "options": "Ref Quota"},
+             "properties": [{"id": "unit", "value": "bytes"}]},
+            {"matcher": {"id": "byName", "options": "Auto-snapshot"},
+             "properties": [{"id": "mappings", "value": [{"type": "value", "options": {
+                 "0": {"text": "no", "color": "orange", "index": 0},
+                 "1": {"text": "yes", "color": "green", "index": 1},
+             }}]}]},
+        ],
+        desc="A quota of 0 means unlimited. Auto-snapshot reflects the "
+             "com.sun:auto-snapshot property that zfs-auto-snapshot honours.",
+    ),
+]))
+
+panels.append(row("Collector Health", 109, collapsed=True, children=[
+    timeseries(
+        "ZFS Collector Duration",
+        [("zfs_metrics_collection_duration_seconds", "collection")],
+        gp(0, 110, 8, 7), unit="s", minval=0,
+        desc="How long collectors/zfs-metrics.py takes. Listing every snapshot is the "
+             "slow part; if this approaches the 1 minute timer interval, raise "
+             "OnUnitActiveSec.",
+    ),
+    timeseries(
+        "Textfile Age",
+        [("time() - node_textfile_mtime_seconds", "{{file}}")],
+        gp(8, 110, 8, 7), unit="s", minval=0,
+        desc="Age of each .prom file. If this climbs without bound the collector has "
+             "stopped writing and every ZFS metric above is stale.",
+    ),
+    timeseries(
+        "Scrape Health", [
+            ('up{job="node"}', "node exporter up"),
+            ("node_textfile_scrape_error", "textfile parse error"),
+        ],
+        gp(16, 110, 8, 7), minval=0,
+        desc="node_textfile_scrape_error goes to 1 if a .prom file is malformed, in "
+             "which case node_exporter drops it entirely.",
+    ),
+]))
+
+dashboard = {
+    "title": "ZFS Pools",
+    "uid": "zfs-pools",
+    "description": "ZFS capacity, health, integrity, snapshot retention, dataset "
+                   "properties and performance. Fed by node_exporter's zfs and systemd "
+                   "collectors plus collectors/zfs-metrics.py from lgtm.nix.",
+    "tags": ["zfs", "storage"],
+    "timezone": "browser",
+    "schemaVersion": 41,
+    "version": 1,
+    "editable": True,
+    "graphTooltip": 1,
+    "refresh": "1m",
+    "time": {"from": "now-24h", "to": "now"},
+    "templating": {"list": [{
+        "name": "pool",
+        "label": "Pool",
+        "type": "query",
+        "datasource": DS,
+        "query": {"qryType": 1, "query": "label_values(zfs_pool_health_code, pool)",
+                  "refId": "PrometheusVariableQueryEditor-VariableQuery"},
+        "refresh": 1,
+        "includeAll": True,
+        "allValue": ".*",
+        "multi": True,
+        "current": {"text": ["All"], "value": ["$__all"]},
+        "options": [],
+        "sort": 1,
+    }]},
+    "annotations": {"list": [{
+        "builtIn": 1, "datasource": {"type": "datasource", "uid": "grafana"},
+        "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts", "type": "dashboard",
+    }]},
+    "panels": panels,
+}
+
+print(json.dumps(dashboard, indent=2))