path: root/dashboards/gen_zfs.py

#!/usr/bin/env python3
"""Generate the "ZFS Pools" Grafana dashboard.

Run this and commit the result:

    python3 dashboards/gen_zfs.py > dashboards/zfs-pools.json

Hand-writing ~2000 lines of Grafana panel JSON invites subtle inconsistencies, so
panels are built from a handful of helpers instead -- every timeseries goes through one
code path and is therefore structurally identical by construction.

The queries here are deliberately limited to metrics we actually produce:

* zfs_pool_* / zfs_dataset_* / zfs_snapshot_*  -- collectors/zfs-metrics.py
* node_zfs_arc_* / node_zfs_zpool_dataset_*    -- node_exporter's zfs collector
* node_systemd_*                               -- node_exporter's systemd collector

If you add a panel, add the metric to the collector first and keep this list honest.
"""

import json

from panels import (DS, OK_BAD, bargauge, gauge, gp, query_var, row, stat, table,
                    timeseries)


STATE_MAP = [{
    "type": "value",
    "options": {
        "0": {"text": "ONLINE", "color": "green", "index": 0},
        "1": {"text": "DEGRADED", "color": "orange", "index": 1},
        "2": {"text": "FAULTED", "color": "red", "index": 2},
        "3": {"text": "OFFLINE", "color": "text", "index": 3},
        "4": {"text": "UNAVAIL", "color": "red", "index": 4},
        "5": {"text": "REMOVED", "color": "purple", "index": 5},
        "6": {"text": "SUSPENDED", "color": "red", "index": 6},
        "7": {"text": "AVAIL", "color": "blue", "index": 7},
        "8": {"text": "UNKNOWN", "color": "text", "index": 8},
    },
}]

BOOL_MAP = [{
    "type": "value",
    "options": {
        "0": {"text": "idle", "color": "green", "index": 0},
        "1": {"text": "RUNNING", "color": "blue", "index": 1},
    },
}]

# Every query on this dashboard is scoped to exactly one machine. The host variable is
# single-select with no "All" on purpose: pool names are not unique across a fleet, and
# the `max by (pool)` reductions and joinByField transformations below would silently
# merge two different pools that happen to share a name.
HOST = 'host=~"$host"'
POOL = f'{HOST}, pool=~"$pool"'
panels = []

# ======================================================================================
# Pool Overview
# ======================================================================================
panels.append(row("Pool Overview", 0))

panels.append(stat(
    "Pool Health", [(f'zfs_pool_health_code{{{POOL}}}', "{{pool}}")],
    gp(0, 1, 4, 4), mappings=STATE_MAP, colormode="background",
    desc="Health as reported by `zpool list -o health`.",
))
panels.append(stat(
    "Data Errors", [(f'sum(zfs_pool_data_errors{{{POOL}}}) or vector(0)', "errors")],
    gp(4, 1, 4, 4), thresholds=OK_BAD, colormode="background",
    desc="Permanent, unrepairable errors from `zpool status`. Anything above zero "
         "means data has been lost and the affected files need restoring.",
))
panels.append(stat(
    "vdev Errors",
    [(f'sum(zfs_pool_vdev_errors_total{{{POOL}}}) or vector(0)', "errors")],
    gp(8, 1, 4, 4), thresholds=OK_BAD, colormode="background",
    desc="Total read + write + checksum errors across all vdevs. Cleared by "
         "`zpool clear`.",
))
panels.append(stat(
    "Last Scrub",
    [(f'time() - max(zfs_pool_last_scrub_end_timestamp_seconds{{{POOL}}})', "ago")],
    gp(12, 1, 4, 4), unit="s", colormode="background",
    thresholds={"mode": "absolute", "steps": [
        {"color": "green", "value": None},
        {"color": "yellow", "value": 35 * 86400},
        {"color": "red", "value": 70 * 86400},
    ]},
    desc="Time since the most recent completed scrub. Turns amber past ~5 weeks, "
         "which is a monthly scrub schedule having missed a run.",
))
panels.append(stat(
    "Scrub / Resilver", [
        (f'max(zfs_pool_scrub_in_progress{{{POOL}}})', "scrub"),
        (f'max(zfs_pool_resilver_in_progress{{{POOL}}})', "resilver"),
    ],
    gp(16, 1, 4, 4), mappings=BOOL_MAP, colormode="background",
))
panels.append(stat(
    "Locked Datasets",
    [(f'count(zfs_dataset_encrypted{{{POOL}}} == 1 and '
      f'zfs_dataset_key_available{{{POOL}}} == 0) or vector(0)', "locked")],
    gp(20, 1, 4, 4), thresholds=OK_BAD, colormode="background",
    desc="Encrypted datasets whose key is not loaded. These are unreadable and are "
         "silently skipped by anything that walks the filesystem, including backups.",
))

panels.append(gauge(
    "Space Used", [(f'zfs_pool_capacity_ratio{{{POOL}}}', "{{pool}}")],
    gp(0, 5, 5, 4),
    desc="How full the pool is: allocated / total. ZFS write performance degrades "
         "sharply past ~80%, and the pool becomes effectively read-only near 100%. "
         "The byte figures behind this are in 'Capacity by Pool' below.",
))
panels.append(gauge(
    "Fragmentation", [(f'zfs_pool_fragmentation_ratio{{{POOL}}}', "{{pool}}")],
    gp(5, 5, 5, 4),
    thresholds={"mode": "absolute", "steps": [
        {"color": "green", "value": None},
        {"color": "yellow", "value": 0.5},
        {"color": "red", "value": 0.75},
    ]},
    desc="Free-space fragmentation.",
))
panels.append(stat(
    "Free Space", [(f'zfs_pool_free_bytes{{{POOL}}}', "{{pool}}")],
    gp(10, 5, 5, 4), unit="bytes",
))
panels.append(stat(
    "Snapshots Overdue",
    [(f'count((time() - zfs_snapshot_newest_timestamp_seconds{{{POOL}}}) > on(label) '
      f'group_left() (2 * zfs_snapshot_interval_seconds{{{HOST}}})) or vector(0)', "overdue")],
    gp(15, 5, 4, 4), thresholds=OK_BAD, colormode="background",
    desc="Dataset/label pairs whose newest snapshot is more than two intervals old — "
         "i.e. the auto-snapshot timer has missed at least one run.",
))
panels.append(stat(
    "Retention Depth",
    [(f'time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{{{POOL}}})', "oldest")],
    gp(19, 5, 5, 4), unit="s",
    desc="Age of the oldest snapshot on the pool — how far back you can actually "
         "restore from.",
))

panels.append(timeseries(
    "Pool Space Used / Free", [
        (f'zfs_pool_allocated_bytes{{{POOL}}}', "{{pool}} used"),
        (f'zfs_pool_free_bytes{{{POOL}}}', "{{pool}} free"),
    ],
    gp(0, 9, 12, 8), unit="bytes", stack=True, minval=0,
    desc="Stacked, so the top of each pool's band is its total size and the split "
         "between the bands is the usage fraction the gauge above shows.",
))
panels.append(timeseries(
    "Fragmentation",
    [(f'zfs_pool_fragmentation_ratio{{{POOL}}}', "{{pool}}")],
    gp(12, 9, 12, 8), unit="percentunit", minval=0, maxval=1,
    desc="Free-space fragmentation — how scattered the pool's remaining free space "
         "is, not how scattered the data is. It rises as a pool fills and stays high "
         "afterwards; only rewriting data (send/recv, or deleting a lot) brings it "
         "down. Kept on its own axis because it is unrelated to space used, and "
         "plotting the two together invites reading one as the other.",
))

panels.append(table(
    "Capacity by Pool", [
        (f'zfs_pool_size_bytes{{{POOL}}}', None),
        (f'max by (pool) (zfs_pool_allocated_bytes{{{POOL}}})', None),
        (f'max by (pool) (zfs_pool_free_bytes{{{POOL}}})', None),
        (f'max by (pool) (zfs_pool_capacity_ratio{{{POOL}}})', None),
        (f'max by (pool) (zfs_pool_fragmentation_ratio{{{POOL}}})', None),
        (f'max by (pool) (zfs_pool_snapshot_used_bytes{{{POOL}}})', None),
        (f'max by (pool) (zfs_pool_dedup_ratio{{{POOL}}})', None),
    ],
    gp(0, 17, 24, 8), keep="pool|Value.*",
    renames={
        "__join__": "pool", "pool": "Pool",
        "Value #A": "Size", "Value #B": "Used", "Value #C": "Free",
        "Value #D": "Used %", "Value #E": "Fragmentation",
        "Value #F": "Snapshots", "Value #G": "Dedup",
    },
    order=["pool", "Value #A", "Value #B", "Value #C", "Value #D", "Value #E",
           "Value #F", "Value #G"],
    sort_by="Used %",
    overrides=[
        # Anchored: byRegexp is a substring match, so a bare "Used" would also
        # capture "Used %" and fight the percent override below.
        {"matcher": {"id": "byRegexp", "options": "^(Size|Used|Free|Snapshots)$"},
         "properties": [{"id": "unit", "value": "bytes"}]},
        {"matcher": {"id": "byName", "options": "Used %"},
         "properties": [
             {"id": "unit", "value": "percentunit"},
             {"id": "max", "value": 1},
             {"id": "thresholds", "value": {"mode": "absolute", "steps": [
                 {"color": "green", "value": None},
                 {"color": "yellow", "value": 0.8},
                 {"color": "red", "value": 0.9},
             ]}},
             {"id": "custom.cellOptions",
              "value": {"type": "gauge", "mode": "gradient"}},
         ]},
        {"matcher": {"id": "byName", "options": "Fragmentation"},
         "properties": [{"id": "unit", "value": "percentunit"}]},
    ],
    desc="Per-pool space in bytes. 'Size' is the raw pool size zpool reports, which "
         "for raidz is before parity overhead — so 'Free' is the number to trust for "
         "how much you can still write. 'Snapshots' is space held only by snapshots.",
))

# ======================================================================================
# Integrity & Errors
# ======================================================================================
panels.append(row("Integrity & Errors", 25))

panels.append(table(
    "vdev State", [(f'zfs_pool_vdev_state_code{{{POOL}}}', None)],
    gp(0, 26, 12, 9), keep="pool|vdev|Value",
    renames={"pool": "Pool", "vdev": "vdev", "Value": "State"},
    mappings=STATE_MAP,
    overrides=[{
        "matcher": {"id": "byName", "options": "State"},
        "properties": [{"id": "custom.cellOptions",
                        "value": {"type": "color-background", "mode": "basic"}}],
    }],
    desc="Every vdev and leaf device in the selected pool(s).",
))
panels.append(timeseries(
    "vdev Error Counters",
    [(f'zfs_pool_vdev_errors_total{{{POOL}}}', "{{pool}}/{{vdev}} {{type}}")],
    gp(12, 26, 12, 9), minval=0,
    custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
    desc="Cumulative per-vdev errors. A counter that climbs is a disk on its way out.",
))

panels.append(table(
    "Scrub & Resilver History", [
        (f'zfs_pool_last_scrub_end_timestamp_seconds{{{POOL}}} * 1000', None),
        (f'zfs_pool_last_scrub_duration_seconds{{{POOL}}}', None),
        (f'zfs_pool_last_scrub_errors{{{POOL}}}', None),
        (f'zfs_pool_last_scrub_repaired_bytes{{{POOL}}}', None),
        # zpool status reports one scan per pool, so a pool whose most recent scan was
        # a resilver has no scrub record at all and would otherwise vanish from this
        # table. The outer join keeps it, with only this column filled.
        (f'zfs_pool_last_resilver_end_timestamp_seconds{{{POOL}}} * 1000', None),
    ],
    gp(0, 35, 12, 7), keep="pool|Value.*",
    renames={
        "__join__": "pool",
        "pool": "Pool",
        "Value #A": "Scrub Finished", "Value #B": "Duration",
        "Value #C": "Errors", "Value #D": "Repaired",
        "Value #E": "Resilver Finished",
    },
    overrides=[
        {"matcher": {"id": "byName", "options": "Scrub Finished"},
         "properties": [{"id": "unit", "value": "dateTimeAsIso"}]},
        {"matcher": {"id": "byName", "options": "Resilver Finished"},
         "properties": [{"id": "unit", "value": "dateTimeAsIso"}]},
        {"matcher": {"id": "byName", "options": "Duration"},
         "properties": [{"id": "unit", "value": "s"}]},
        {"matcher": {"id": "byName", "options": "Repaired"},
         "properties": [{"id": "unit", "value": "bytes"}]},
        {"matcher": {"id": "byName", "options": "Errors"},
         "properties": [
             {"id": "custom.cellOptions",
              "value": {"type": "color-background", "mode": "basic"}},
             {"id": "thresholds", "value": OK_BAD},
         ]},
    ],
    desc="Result of the last completed scrub per pool. Errors here are blocks that "
         "could not be repaired from redundancy.",
))
panels.append(timeseries(
    "Pool Data Errors", [(f'zfs_pool_data_errors{{{POOL}}}', "{{pool}}")],
    gp(12, 35, 12, 7), minval=0,
    custom={"lineWidth": 2, "fillOpacity": 10, "showPoints": "auto"},
    desc="Permanent data errors over time.",
))

# ======================================================================================
# Snapshots & Retention
# ======================================================================================
panels.append(row("Snapshots & Retention", 42))

panels.append(bargauge(
    "Time Since Last Snapshot",
    [(f'time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{{{POOL}}})',
      "{{label}}")],
    gp(0, 43, 8, 8), unit="s",
    thresholds={"mode": "absolute", "steps": [
        {"color": "green", "value": None},
        {"color": "yellow", "value": 2 * 3600},
        {"color": "red", "value": 2 * 86400},
    ]},
    desc="Age of the newest snapshot of each label. Compare against the label's own "
         "cadence — 'frequent' is every 15 minutes, 'monthly' every month, so the "
         "colour thresholds here are only a coarse hint.",
))
panels.append(bargauge(
    "Snapshots vs Retention Target",
    [(f'min by (label) (zfs_snapshot_count{{{POOL}}} / on(label) group_left() '
      f'zfs_snapshot_retention_target{{{HOST}}})', "{{label}}")],
    gp(8, 43, 8, 8), unit="percentunit", maxv=1.2,
    thresholds={"mode": "absolute", "steps": [
        {"color": "red", "value": None},
        {"color": "yellow", "value": 0.5},
        {"color": "green", "value": 0.9},
    ]},
    desc="Worst dataset's snapshot count as a fraction of the configured "
         "services.zfs.autoSnapshot retention. 100% means the full set is on disk; a "
         "freshly-created dataset legitimately sits below target until it fills up.",
))
panels.append(bargauge(
    "Oldest Snapshot",
    [(f'time() - min by (label) '
      f'(zfs_pool_snapshot_oldest_timestamp_seconds{{{POOL}}})', "{{label}}")],
    gp(16, 43, 8, 8), unit="s",
    desc="How far back each label reaches.",
))

panels.append(timeseries(
    "Snapshot Count by Label",
    [(f'sum by (label) (zfs_snapshot_count{{{POOL}}})', "{{label}}")],
    gp(0, 51, 12, 8), minval=0,
    desc="Total snapshots per label. A steady sawtooth is healthy rotation; a "
         "monotonic decline means snapshots are expiring but not being replaced.",
))
panels.append(timeseries(
    "Snapshot Space by Label",
    [(f'sum by (label) (zfs_pool_snapshot_used_bytes{{{POOL}}})', "{{label}}")],
    gp(12, 51, 12, 8), unit="bytes", stack=True, minval=0,
    desc="Space held down by snapshots — the cost of your retention policy.",
))

panels.append(stat(
    "Snapshot / Scrub Job Failures",
    [(f'sum(node_systemd_unit_state{{{HOST}, name=~"zfs-(snapshot|scrub).*", '
      f'state="failed"}}) or vector(0)', "failed units")],
    gp(0, 59, 6, 6), thresholds=OK_BAD, colormode="background",
    desc="systemd units for the auto-snapshot and auto-scrub timers currently in a "
         "failed state.",
))
panels.append(bargauge(
    "Time Since Timer Last Fired",
    [(f'time() - node_systemd_timer_last_trigger_seconds{{{HOST}, '
      f'name=~"zfs-.*"}}', "{{name}}")],
    gp(6, 59, 18, 6), unit="s",
    desc="When each zfs-* systemd timer last triggered. This is the job side of the "
         "picture; the snapshot ages above are the result side.",
))

# ======================================================================================
# Datasets
# ======================================================================================
panels.append(row("Datasets", 65))

panels.append(timeseries(
    "Dataset Usage Over Time",
    [(f'zfs_dataset_used_bytes{{{POOL}}}', "{{dataset}}")],
    gp(0, 66, 24, 9), unit="bytes", minval=0,
    custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
    desc="Space each dataset is responsible for, including its snapshots and "
         "descendants — so a parent's line contains its children's. Unstacked and "
         "unfilled so a single dataset's growth is readable against the others rather "
         "than riding on top of them. The legend carries last and max per dataset; the "
         "instant breakdown is in 'Capacity by Dataset' below.",
))

panels.append(table(
    "Capacity by Dataset", [
        # Only this query keeps `pool` as its own column; the rest are reduced to the
        # join key so joinByField has nothing to collide on. `dataset` is the full
        # path ("ingens/media"), so it is unique on its own.
        (f'zfs_dataset_used_bytes{{{POOL}}}', None),
        (f'max by (dataset) (zfs_dataset_used_by_dataset_bytes{{{POOL}}})', None),
        (f'max by (dataset) (zfs_dataset_used_by_snapshots_bytes{{{POOL}}})', None),
        (f'max by (dataset) (zfs_dataset_referenced_bytes{{{POOL}}})', None),
        (f'max by (dataset) (zfs_dataset_logical_used_bytes{{{POOL}}})', None),
        (f'max by (dataset) (zfs_dataset_available_bytes{{{POOL}}})', None),
        (f'max by (dataset) (zfs_dataset_compression_ratio{{{POOL}}})', None),
        # Quota is 0 when unset, which would read as "no space allowed"; blank it.
        (f'max by (dataset) (zfs_dataset_quota_bytes{{{POOL}}} > 0)', None),
    ],
    gp(0, 75, 24, 8), keep="pool|dataset|Value.*",
    renames={
        "__join__": "dataset", "dataset": "Dataset", "pool": "Pool",
        "Value #A": "Used", "Value #B": "Own Data", "Value #C": "Snapshots",
        "Value #D": "Referenced", "Value #E": "Logical", "Value #F": "Available",
        "Value #G": "Ratio", "Value #H": "Quota",
    },
    order=["pool", "dataset", "Value #A", "Value #B", "Value #C", "Value #D",
           "Value #E", "Value #F", "Value #G", "Value #H"],
    sort_by="Used",
    overrides=[
        {"matcher": {"id": "byRegexp",
                     "options": "^(Used|Own Data|Snapshots|Referenced|Logical|"
                                "Available|Quota)$"},
         "properties": [{"id": "unit", "value": "bytes"}]},
        {"matcher": {"id": "byName", "options": "Quota"},
         "properties": [{"id": "noValue", "value": "none"}]},
        {"matcher": {"id": "byName", "options": "Ratio"},
         "properties": [{"id": "decimals", "value": 2}]},
    ],
    desc="Per-dataset space, separate from the per-pool figures above. 'Used' is the "
         "total a dataset is responsible for; 'Own Data' and 'Snapshots' are the two "
         "halves of that. 'Logical' is the size before compression, so Logical / "
         "Referenced is where the compression ratio comes from. 'Available' is shared "
         "with every sibling dataset unless a quota says otherwise.",
))

panels.append(table(
    "Compression",
    [(f'zfs_dataset_compression_ratio{{{POOL}}} * on(pool, dataset) '
      f'group_left(compression) zfs_dataset_compression{{{POOL}}}', None)],
    gp(0, 83, 12, 8), keep="pool|dataset|compression|Value",
    order=["pool", "dataset", "compression", "Value"],
    renames={"pool": "Pool", "dataset": "Dataset",
             "compression": "Algorithm", "Value": "Ratio"},
    sort_by="Ratio",
    desc="group_left() attaches the algorithm label from the info metric onto the "
         "ratio, so this needs only one query and no join transformation.",
))
panels.append(table(
    "Encryption",
    [(f'zfs_dataset_keystatus_code{{{POOL}}} * on(pool, dataset) '
      f'group_left(encryption) zfs_dataset_encryption{{{POOL}}}', None)],
    gp(12, 83, 12, 8), keep="pool|dataset|encryption|Value",
    order=["pool", "dataset", "encryption", "Value"],
    renames={"pool": "Pool", "dataset": "Dataset",
             "encryption": "Suite", "Value": "Key"},
    mappings=[{"type": "value", "options": {
        "0": {"text": "—", "color": "text", "index": 0},
        "1": {"text": "unlocked", "color": "green", "index": 1},
        "2": {"text": "LOCKED", "color": "red", "index": 2},
        "3": {"text": "unknown", "color": "orange", "index": 3},
    }}],
    overrides=[{
        "matcher": {"id": "byName", "options": "Key"},
        "properties": [{"id": "custom.cellOptions",
                        "value": {"type": "color-background", "mode": "basic"}}],
    }],
    desc="Encryption suite per dataset and the state of its key. Datasets that are "
         "not encrypted show suite 'off' and a dash — only an encrypted dataset can "
         "be locked.",
))

# ======================================================================================
# Performance
# ======================================================================================
panels.append(row("Performance", 91))

panels.append(timeseries(
    "Dataset Throughput", [
        (f'rate(node_zfs_zpool_dataset_nread{{{HOST}, zpool=~"$pool"}}[$__rate_interval])',
         "{{zpool}}/{{dataset}} read"),
        (f'-rate(node_zfs_zpool_dataset_nwritten{{{HOST}, zpool=~"$pool"}}[$__rate_interval])',
         "{{zpool}}/{{dataset}} write"),
    ],
    gp(0, 92, 12, 8), unit="Bps",
    desc="Per-dataset throughput from node_exporter's objset kstats. Writes are drawn "
         "negative.",
))
panels.append(timeseries(
    "Dataset IOPS", [
        (f'rate(node_zfs_zpool_dataset_reads{{{HOST}, zpool=~"$pool"}}[$__rate_interval])',
         "{{zpool}}/{{dataset}} read"),
        (f'-rate(node_zfs_zpool_dataset_writes{{{HOST}, zpool=~"$pool"}}[$__rate_interval])',
         "{{zpool}}/{{dataset}} write"),
    ],
    gp(12, 92, 12, 8), unit="iops",
    desc="Writes are drawn negative.",
))

panels.append(timeseries(
    "ARC Size", [
        (f"node_zfs_arc_size{{{HOST}}}", "size"),
        (f"node_zfs_arc_c_max{{{HOST}}}", "target max (c_max)"),
        (f"node_zfs_arc_c_min{{{HOST}}}", "target min (c_min)"),
    ],
    gp(0, 100, 8, 8), unit="bytes", minval=0,
))
panels.append(timeseries(
    "ARC Hit Ratio",
    [(f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval]) / clamp_min("
      f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval]) + "
      f"rate(node_zfs_arc_misses{{{HOST}}}[$__rate_interval]), 1)", "hit ratio")],
    gp(8, 100, 8, 8), unit="percentunit", minval=0,
    desc="clamp_min avoids a divide-by-zero while the pool is idle.",
))
panels.append(timeseries(
    "ARC Hits / Misses", [
        (f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval])", "hits"),
        (f"rate(node_zfs_arc_misses{{{HOST}}}[$__rate_interval])", "misses"),
    ],
    gp(16, 100, 8, 8), unit="ops", minval=0,
))

# ======================================================================================
# Collapsed: advanced / diagnostic
# ======================================================================================
panels.append(row("Pool Internals", 108, collapsed=True, children=[
    timeseries(
        "Reclaim & Leaks", [
            (f'zfs_pool_freeing_bytes{{{POOL}}}', "{{pool}} freeing"),
            (f'zfs_pool_leaked_bytes{{{POOL}}}', "{{pool}} leaked"),
            (f'zfs_pool_checkpoint_bytes{{{POOL}}}', "{{pool}} checkpoint"),
        ],
        gp(0, 109, 8, 7), unit="bytes", minval=0,
        desc="`freeing` is space still being reclaimed in the background after a "
             "delete. `leaked` should always be zero.",
    ),
    timeseries(
        "Dedup Ratio", [(f'zfs_pool_dedup_ratio{{{POOL}}}', "{{pool}}")],
        gp(8, 109, 8, 7), minval=0,
        desc="1.00 unless deduplication is enabled, which it usually should not be.",
    ),
    timeseries(
        "Pool Health History", [
            (f'zfs_pool_health_code{{{POOL}}}', "{{pool}}"),
        ],
        # Bounded to the state-code range so the y-axis reads as states, not 0-100.
        gp(16, 109, 8, 7), mappings=STATE_MAP, minval=0, maxval=8,
        custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"},
        desc="Health code over time — useful for spotting a pool that briefly "
             "degraded and recovered. zfs_pool_scan_state is also collected if you "
             "want to plot scrub/resilver phases alongside it.",
    ),
    table(
        "Dataset Properties", [
            (f'zfs_dataset_recordsize_bytes{{{POOL}}}', None),
            (f'zfs_dataset_quota_bytes{{{POOL}}}', None),
            (f'zfs_dataset_refquota_bytes{{{POOL}}}', None),
            (f'zfs_dataset_autosnapshot_enabled{{{POOL}}}', None),
        ],
        gp(0, 116, 24, 8), keep="dataset|Value.*",
        renames={
            "__join__": "dataset", "dataset": "Dataset",
            "Value #A": "Record Size", "Value #B": "Quota",
            "Value #C": "Ref Quota", "Value #D": "Auto-snapshot",
        },
        overrides=[
            {"matcher": {"id": "byName", "options": "Record Size"},
             "properties": [{"id": "unit", "value": "bytes"}]},
            {"matcher": {"id": "byName", "options": "Quota"},
             "properties": [{"id": "unit", "value": "bytes"}]},
            {"matcher": {"id": "byName", "options": "Ref Quota"},
             "properties": [{"id": "unit", "value": "bytes"}]},
            {"matcher": {"id": "byName", "options": "Auto-snapshot"},
             "properties": [{"id": "mappings", "value": [{"type": "value", "options": {
                 "0": {"text": "no", "color": "orange", "index": 0},
                 "1": {"text": "yes", "color": "green", "index": 1},
             }}]}]},
        ],
        desc="A quota of 0 means unlimited. Auto-snapshot reflects the "
             "com.sun:auto-snapshot property that zfs-auto-snapshot honours.",
    ),
]))

panels.append(row("Collector Health", 109, collapsed=True, children=[
    timeseries(
        "ZFS Collector Duration",
        [(f"zfs_metrics_collection_duration_seconds{{{HOST}}}", "collection")],
        gp(0, 110, 8, 7), unit="s", minval=0,
        desc="How long collectors/zfs-metrics.py takes. Listing every snapshot is the "
             "slow part; if this approaches the 1 minute timer interval, raise "
             "OnUnitActiveSec.",
    ),
    timeseries(
        "Textfile Age",
        [(f"time() - node_textfile_mtime_seconds{{{HOST}}}", "{{file}}")],
        gp(8, 110, 8, 7), unit="s", minval=0,
        desc="Age of each .prom file. If this climbs without bound the collector has "
             "stopped writing and every ZFS metric above is stale.",
    ),
    timeseries(
        "Scrape Health", [
            ('up{job="node"}', "node exporter up"),
            (f"node_textfile_scrape_error{{{HOST}}}", "textfile parse error"),
        ],
        gp(16, 110, 8, 7), minval=0,
        desc="node_textfile_scrape_error goes to 1 if a .prom file is malformed, in "
             "which case node_exporter drops it entirely.",
    ),
]))

dashboard = {
    "title": "ZFS Pools",
    "uid": "zfs-pools",
    "description": "ZFS capacity, health, integrity, snapshot retention, dataset "
                   "properties and performance. Fed by node_exporter's zfs and systemd "
                   "collectors plus collectors/zfs-metrics.py from lgtm.nix.",
    "tags": ["zfs", "storage"],
    "timezone": "browser",
    "schemaVersion": 41,
    "version": 1,
    "editable": True,
    "graphTooltip": 1,
    "refresh": "1m",
    "time": {"from": "now-24h", "to": "now"},
    "templating": {"list": [
        # One machine at a time: not multi-select and no "All". See the comment on HOST
        # above for why two hosts at once would silently merge same-named pools.
        query_var("host", "label_values(zfs_pool_health_code, host)",
                  label="Host", multi=False, include_all=False),
        # Pools are listed for the selected host only, so switching hosts re-populates
        # the picker rather than offering pools that do not exist on this machine.
        query_var("pool", 'label_values(zfs_pool_health_code{host=~"$host"}, pool)',
                  label="Pool"),
    ]},
    "annotations": {"list": [{
        "builtIn": 1, "datasource": {"type": "datasource", "uid": "grafana"},
        "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts", "type": "dashboard",
    }]},
    "panels": panels,
}

print(json.dumps(dashboard, indent=2))