initial commit

1 files changed, 584 insertions, 0 deletions

diff --git a/dashboards/gen_smart.py b/dashboards/gen_smart.py
new file mode 100644
index 0000000..2cc0568
--- /dev/null
+++ b/dashboards/gen_smart.py
@@ -0,0 +1,584 @@
+#!/usr/bin/env python3
+"""Generate the "Drive Health (SMART)" Grafana dashboard.
+
+Run this and commit the result:
+
+    python3 dashboards/gen_smart.py > dashboards/smart.json
+
+A CrystalDiskInfo-style view: a Good/Caution/Bad verdict per drive, the full raw SMART
+attribute table, and self-test history.
+
+The fleet is mixed, so there are two repeating sections:
+
+* SATA drives expose the classic ATA attribute table (value / worst / thresh / raw).
+* SAS drives expose none of it. Their health lives in SCSI error counters instead --
+  grown defect list, uncorrected errors, corrected-error breakdown.
+
+Drives are routed into the right section by *metric presence* rather than by the
+`protocol` label, which is more robust: only ATA drives have smartctl_device_attribute,
+and only SCSI drives have smartctl_scsi_grown_defect_list.
+
+Metrics used:
+
+* smartctl_*                 -- prometheus-community/smartctl_exporter
+* smartctl:health_code etc.  -- recording rules in rules/smartctl.yml
+* smart_selftest_*           -- collectors/smart-selftest.py
+"""
+
+import json
+
+from panels import (OK_BAD, dashboard, gp, query_var, row, stat, table,
+                    timeseries)
+
+# Matches the verdict encoding in rules/smartctl.yml.
+HEALTH_MAP = [{
+    "type": "value",
+    "options": {
+        "0": {"text": "GOOD", "color": "green", "index": 0},
+        "1": {"text": "CAUTION", "color": "orange", "index": 1},
+        "2": {"text": "BAD", "color": "red", "index": 2},
+    },
+}]
+
+HEALTH_THRESHOLDS = {"mode": "absolute", "steps": [
+    {"color": "green", "value": None},
+    {"color": "orange", "value": 1},
+    {"color": "red", "value": 2},
+]}
+
+PASS_MAP = [{
+    "type": "value",
+    "options": {
+        "0": {"text": "FAILED", "color": "red", "index": 0},
+        "1": {"text": "passed", "color": "green", "index": 1},
+    },
+}]
+
+ATA = 'device=~"$ata_device"'
+SAS = 'device=~"$sas_device"'
+
+# Splits "sde · GOOD · 6.0 TB · SAS 7.2K · HGST HUS726060AL5210 · NAHPJS45" into a
+# selectable value (the leading device name) and display text (the whole line). The
+# groups nest rather than sit side by side, which is what lets text be a superset of
+# value; matching on whitespace keeps it independent of the separator the collector
+# chose.
+ROW_REGEX = r"/^(?<text>(?<value>\S+)\s.*)$/"
+
+
+def power_on(sel=""):
+    """Best available powered-on seconds per drive, in seconds.
+
+    On ATA, smartctl reads power_on_time straight from attribute 9 and it is reliable.
+    On SAS there is no such attribute: smartctl sources power_on_time from the
+    Background Scan Results log page, whose "Accumulated power on time" counter is
+    maintained by the background-scan subsystem rather than being the drive's lifetime
+    meter. On the HGST SAS drives here it reads a few thousand hours while the drives'
+    own self-test logs record tens of thousands.
+
+    The self-test log's lifetime-hours column is stamped by the drive at the moment
+    each test ran, so it can never exceed the true powered-on figure. Taking whichever
+    source is larger is therefore never an over-estimate, and recovers a sane number on
+    drives whose power-on counter under-counts or was reset.
+    """
+    braces = f"{{{sel}}}" if sel else ""
+    return (f'max by (device) (smartctl_device_power_on_seconds{braces} '
+            f'or (smart_selftest_last_lifetime_hours{braces} * 3600))')
+
+
+def drive_info(title, sel, pos, label_cols, renames, desc=""):
+    """One-row identity card for a drive: what it is, not how it is doing.
+
+    smartctl_device is an info metric whose value is always 1 and whose payload is
+    entirely in its labels, so the value column gets dropped. Rotation rate and
+    capacity are joined in as real numbers; the join is an outer one so an SSD, which
+    reports no rotation rate at all, still shows up with a blank RPM cell.
+    """
+    return table(
+        title, [
+            (f'smartctl_device{{{sel}}}', None),
+            (f'max by (device) (smartctl_device_rotation_rate{{{sel}}})', None),
+            (f'max by (device) (smartctl_device_capacity_bytes{{{sel}}})', None),
+        ],
+        pos,
+        keep="device|" + "|".join(label_cols) + "|Value.*",
+        renames={"__join__": "device", "device": "Device",
+                 "Value #B": "RPM", "Value #C": "Capacity", **renames},
+        exclude=["Value #A"],
+        order=["device", *label_cols, "Value #B", "Value #C"],
+        overrides=[
+            {"matcher": {"id": "byName", "options": "Capacity"},
+             "properties": [{"id": "unit", "value": "bytes"}]},
+            {"matcher": {"id": "byName", "options": "RPM"},
+             "properties": [{"id": "unit", "value": "rotrpm"},
+                            {"id": "noValue", "value": "SSD"}]},
+        ],
+        desc=desc,
+    )
+
+
+panels = []
+
+# ======================================================================================
+# Fleet Overview
+# ======================================================================================
+panels.append(row("Fleet Overview", 0))
+
+panels.append(stat(
+    "Healthy", [('count(smartctl:health_code == 0) or vector(0)', "drives")],
+    gp(0, 1, 4, 4), colormode="background",
+    thresholds={"mode": "absolute", "steps": [{"color": "green", "value": None}]},
+    desc="Drives with no failing attribute, no SCSI uncorrected errors and a passing "
+         "SMART self-assessment.",
+))
+panels.append(stat(
+    "Caution", [('count(smartctl:health_code == 1) or vector(0)', "drives")],
+    gp(4, 1, 4, 4), colormode="background",
+    thresholds={"mode": "absolute", "steps": [
+        {"color": "green", "value": None}, {"color": "orange", "value": 1}]},
+    desc="Reallocated / pending / uncorrectable sectors on SATA, or a non-empty grown "
+         "defect list on SAS. Not failing yet — worth watching the trend.",
+))
+panels.append(stat(
+    "Failing", [('count(smartctl:health_code == 2) or vector(0)', "drives")],
+    gp(8, 1, 4, 4), colormode="background", thresholds=OK_BAD,
+    desc="SMART self-assessment failed, an attribute is at or below its threshold, or "
+         "the drive reported errors it could not correct. Replace these.",
+))
+panels.append(stat(
+    "Drives", [('count(smartctl_device_smart_status)', "total")],
+    gp(12, 1, 4, 4),
+))
+panels.append(stat(
+    "Hottest Drive",
+    [('max(smartctl_device_temperature{temperature_type="current"})', "max")],
+    gp(16, 1, 4, 4), unit="celsius", colormode="background",
+    thresholds={"mode": "absolute", "steps": [
+        {"color": "green", "value": None},
+        {"color": "yellow", "value": 45},
+        {"color": "red", "value": 55},
+    ]},
+))
+panels.append(stat(
+    "Longest Powered On", [(f'max({power_on()})', "max")],
+    gp(20, 1, 4, 4), unit="s",
+))
+
+panels.append(table(
+    "All Drives", [
+        # group_left carries the descriptive labels from the info metric onto the
+        # verdict, so this one query supplies device, model, serial and protocol.
+        ('smartctl:health_code * on(device) group_left(model_name, serial_number, '
+         'protocol) smartctl_device', None),
+        ('max by (device) (smartctl_device_temperature{temperature_type="current"})',
+         None),
+        (power_on(), None),
+        ('max by (device) (smartctl_device_capacity_bytes)', None),
+    ],
+    gp(0, 5, 16, 10), keep="device|model_name|serial_number|protocol|Value.*",
+    renames={
+        "__join__": "device", "device": "Device", "model_name": "Model",
+        "serial_number": "Serial", "protocol": "Bus",
+        "Value #A": "Health", "Value #B": "Temp",
+        "Value #C": "Powered On", "Value #D": "Capacity",
+    },
+    order=["device", "model_name", "serial_number", "protocol",
+           "Value #A", "Value #B", "Value #C", "Value #D"],
+    sort_by="Health",
+    overrides=[
+        {"matcher": {"id": "byName", "options": "Health"},
+         "properties": [
+             {"id": "mappings", "value": HEALTH_MAP},
+             {"id": "thresholds", "value": HEALTH_THRESHOLDS},
+             {"id": "custom.cellOptions",
+              "value": {"type": "color-background", "mode": "basic"}},
+         ]},
+        {"matcher": {"id": "byName", "options": "Temp"},
+         "properties": [{"id": "unit", "value": "celsius"}]},
+        {"matcher": {"id": "byName", "options": "Powered On"},
+         "properties": [{"id": "unit", "value": "s"}]},
+        {"matcher": {"id": "byName", "options": "Capacity"},
+         "properties": [{"id": "unit", "value": "bytes"}]},
+    ],
+    desc="Every drive, worst first. Health is computed by rules/smartctl.yml.",
+))
+
+panels.append(timeseries(
+    "Temperatures",
+    [('smartctl_device_temperature{temperature_type="current"}', "{{device}}")],
+    gp(16, 5, 8, 10), unit="celsius",
+    desc="Sustained operation above ~45C measurably shortens drive life.",
+))
+
+# ======================================================================================
+# SATA — repeats once per ATA drive
+# ======================================================================================
+panels.append(row("${ata_device:text}", 15, repeat="ata_device"))
+
+panels.append(stat(
+    "Health", [(f'smartctl:health_code{{{ATA}}}', "{{device}}")],
+    gp(0, 16, 6, 4), mappings=HEALTH_MAP, thresholds=HEALTH_THRESHOLDS,
+    colormode="background",
+))
+panels.append(stat(
+    "Temperature",
+    [(f'smartctl_device_temperature{{{ATA}, temperature_type="current"}}', "temp")],
+    gp(6, 16, 3, 4), unit="celsius",
+    thresholds={"mode": "absolute", "steps": [
+        {"color": "green", "value": None},
+        {"color": "yellow", "value": 45},
+        {"color": "red", "value": 55},
+    ]},
+))
+panels.append(stat(
+    "Powered On", [(power_on(ATA), "hours")],
+    gp(9, 16, 3, 4), unit="s",
+))
+panels.append(stat(
+    "Capacity", [(f'smartctl_device_capacity_bytes{{{ATA}}}', "bytes")],
+    gp(12, 16, 3, 4), unit="bytes",
+))
+panels.append(stat(
+    "Power Cycles", [(f'smartctl_device_power_cycle_count{{{ATA}}}', "cycles")],
+    gp(15, 16, 3, 4),
+))
+panels.append(stat(
+    "Reallocated Sectors",
+    [(f'smartctl_device_attribute{{{ATA}, attribute_id="5", '
+      f'attribute_value_type="raw"}}', "sectors")],
+    gp(18, 16, 3, 4), thresholds=OK_BAD, colormode="background",
+    desc="Attribute 5 raw. Sectors the drive has already retired. Any growth here is "
+         "the single most predictive sign of impending failure.",
+))
+panels.append(stat(
+    "Pending Sectors",
+    [(f'smartctl_device_attribute{{{ATA}, attribute_id="197", '
+      f'attribute_value_type="raw"}}', "sectors")],
+    gp(21, 16, 3, 4), thresholds=OK_BAD, colormode="background",
+    desc="Attribute 197 raw. Sectors that failed to read and are waiting to be "
+         "reallocated on next write. Often precedes reallocations.",
+))
+
+panels.append(drive_info(
+    "Drive — $ata_device", ATA, gp(0, 20, 24, 4),
+    label_cols=["model_family", "model_name", "serial_number", "firmware_version",
+                "interface", "sata_version", "form_factor"],
+    renames={"model_family": "Family", "model_name": "Model",
+             "serial_number": "Serial", "firmware_version": "Firmware",
+             "interface": "Interface", "sata_version": "SATA", "form_factor": "Form"},
+    desc="Identity of this drive. RPM shows SSD when the drive reports no rotation "
+         "rate.",
+))
+
+panels.append(table(
+    "SMART Attributes — $ata_device", [
+        # Only this query keeps the descriptive labels; the rest are reduced to the
+        # join key so nothing collides in joinByField.
+        (f'smartctl_device_attribute{{{ATA}, attribute_value_type="value"}}', None),
+        (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
+         f'attribute_value_type="worst"}})', None),
+        (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
+         f'attribute_value_type="thresh"}})', None),
+        (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
+         f'attribute_value_type="raw"}})', None),
+        # How much headroom is left before this attribute trips its own failure
+        # threshold. Zero or below is what SMART calls a failure.
+        (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
+         f'attribute_value_type="value"}} - on(device, attribute_id) '
+         f'smartctl_device_attribute{{{ATA}, attribute_value_type="thresh"}})', None),
+    ],
+    gp(0, 24, 24, 12), keep="attribute_id|attribute_name|Value.*",
+    renames={
+        "__join__": "attribute_id", "attribute_id": "ID",
+        "attribute_name": "Attribute",
+        "Value #A": "Current", "Value #B": "Worst", "Value #C": "Threshold",
+        "Value #D": "Raw", "Value #E": "Margin",
+    },
+    order=["attribute_id", "attribute_name",
+           "Value #A", "Value #B", "Value #C", "Value #D", "Value #E"],
+    sort_by="Attribute", sort_desc=False,
+    overrides=[
+        # Margin is the honest place to highlight trouble: a blanket threshold on Raw
+        # would flag Power_On_Hours, which is enormous and perfectly healthy.
+        {"matcher": {"id": "byName", "options": "Margin"},
+         "properties": [
+             {"id": "thresholds", "value": {"mode": "absolute", "steps": [
+                 {"color": "red", "value": None},
+                 {"color": "orange", "value": 1},
+                 {"color": "yellow", "value": 10},
+                 {"color": "green", "value": 25},
+             ]}},
+             {"id": "custom.cellOptions",
+              "value": {"type": "color-background", "mode": "basic"}},
+         ]},
+    ],
+    desc="Every attribute the drive reports, with its raw value. Current/Worst are "
+         "normalised 0-253 scores (higher is better); Threshold is where the vendor "
+         "declares failure; Margin is Current minus Threshold. ID sorts as text, so "
+         "the table is ordered by attribute name.",
+))
+
+# ======================================================================================
+# SAS — repeats once per SCSI drive
+# ======================================================================================
+panels.append(row("${sas_device:text}", 36, repeat="sas_device"))
+
+panels.append(stat(
+    "Health", [(f'smartctl:health_code{{{SAS}}}', "{{device}}")],
+    gp(0, 37, 6, 4), mappings=HEALTH_MAP, thresholds=HEALTH_THRESHOLDS,
+    colormode="background",
+))
+panels.append(stat(
+    "Temperature",
+    [(f'smartctl_device_temperature{{{SAS}, temperature_type="current"}}', "temp")],
+    gp(6, 37, 3, 4), unit="celsius",
+    thresholds={"mode": "absolute", "steps": [
+        {"color": "green", "value": None},
+        {"color": "yellow", "value": 45},
+        {"color": "red", "value": 55},
+    ]},
+))
+panels.append(stat(
+    "Powered On", [(power_on(SAS), "hours")],
+    gp(9, 37, 3, 4), unit="s",
+    desc="SAS drives have no ATA attribute 9, so smartctl reports power_on_time from "
+         "the Background Scan Results log page — a counter the background-scan "
+         "subsystem maintains, not the drive's lifetime meter, and one that commonly "
+         "reads far too low. This tile takes the larger of that counter and the "
+         "lifetime-hours stamp in the drive's own self-test log, which is why it can "
+         "exceed what `smartctl -x` prints as 'Accumulated power on time'.",
+))
+panels.append(stat(
+    "Capacity", [(f'smartctl_device_capacity_bytes{{{SAS}}}', "bytes")],
+    gp(12, 37, 3, 4), unit="bytes",
+))
+panels.append(stat(
+    "Link Speed",
+    [(f'smartctl_device_interface_speed{{{SAS}, speed_type="current"}} * 1000', "bps")],
+    gp(15, 37, 3, 4), unit="bps",
+    desc="Negotiated SAS link rate. A drive that has fallen back to a lower rate is "
+         "worth investigating — usually a cable or expander.",
+))
+panels.append(stat(
+    "Grown Defects", [(f'smartctl_scsi_grown_defect_list{{{SAS}}}', "defects")],
+    gp(18, 37, 3, 4), thresholds=OK_BAD, colormode="background",
+    desc="Sectors reallocated since the drive left the factory — the SAS equivalent "
+         "of ATA attribute 5. A stable non-zero count is tolerable; growth is not.",
+))
+panels.append(stat(
+    "Uncorrected Errors",
+    [(f'smartctl_read_total_uncorrected_errors{{{SAS}}} + '
+      f'smartctl_write_total_uncorrected_errors{{{SAS}}}', "errors")],
+    gp(21, 37, 3, 4), thresholds=OK_BAD, colormode="background",
+    desc="Errors the drive could not correct with ECC or retries, which means data "
+         "was lost. Should be zero.",
+))
+
+panels.append(drive_info(
+    "Drive — $sas_device", SAS, gp(0, 41, 24, 4),
+    label_cols=["scsi_vendor", "model_name", "serial_number", "scsi_revision",
+                "scsi_version", "protocol", "form_factor"],
+    renames={"scsi_vendor": "Vendor", "model_name": "Model",
+             "serial_number": "Serial", "scsi_revision": "Revision",
+             "scsi_version": "SCSI", "protocol": "Bus", "form_factor": "Form"},
+    desc="Identity of this drive. SAS drives leave firmware_version empty and report "
+         "their revision in scsi_revision instead.",
+))
+
+panels.append(timeseries(
+    "Corrected Errors — $sas_device", [
+        (f'smartctl_read_errors_corrected_by_eccfast{{{SAS}}}', "read ECC fast"),
+        (f'smartctl_read_errors_corrected_by_eccdelayed{{{SAS}}}', "read ECC delayed"),
+        (f'smartctl_read_errors_corrected_by_rereads_rewrites{{{SAS}}}', "read retries"),
+        (f'smartctl_write_errors_corrected_by_eccfast{{{SAS}}}', "write ECC fast"),
+        (f'smartctl_write_errors_corrected_by_eccdelayed{{{SAS}}}',
+         "write ECC delayed"),
+        (f'smartctl_write_errors_corrected_by_rereads_rewrites{{{SAS}}}',
+         "write retries"),
+    ],
+    gp(0, 45, 12, 8), minval=0,
+    desc="Errors the drive fixed on its own. ECC-fast corrections are routine and "
+         "grow constantly. Delayed corrections and especially rewrites/retries "
+         "climbing is the signal worth watching.",
+))
+panels.append(timeseries(
+    "Defects & Uncorrected — $sas_device", [
+        (f'smartctl_scsi_grown_defect_list{{{SAS}}}', "grown defects"),
+        (f'smartctl_read_total_uncorrected_errors{{{SAS}}}', "read uncorrected"),
+        (f'smartctl_write_total_uncorrected_errors{{{SAS}}}', "write uncorrected"),
+    ],
+    gp(12, 45, 12, 8), minval=0,
+    custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+    desc="Flat lines are good. Any upward step here is a real event.",
+))
+
+# ======================================================================================
+# Self-tests
+# ======================================================================================
+panels.append(row("Self-Tests", 53))
+
+panels.append(stat(
+    "Drives Never Tested",
+    [('count(smartctl_device_smart_status) - '
+      'count(count by (device) (smart_selftest_last_lifetime_hours)) or '
+      'count(smartctl_device_smart_status)', "drives")],
+    gp(0, 54, 4, 5), thresholds=OK_BAD, colormode="background",
+    desc="Drives with no self-test log entries at all. smartd only runs tests if its "
+         "config includes an -s schedule; merely enabling the service does not.",
+))
+panels.append(stat(
+    "Failed Self-Tests",
+    [('count(smart_selftest_last_passed == 0) or vector(0)', "tests")],
+    gp(4, 54, 4, 5), thresholds=OK_BAD, colormode="background",
+))
+panels.append(stat(
+    "Oldest Long Test",
+    [(f'max({power_on()} / 3600 - on(device) group_right() '
+      f'smart_selftest_last_lifetime_hours{{test_type="extended"}})', "hours ago")],
+    gp(8, 54, 4, 5), unit="h",
+    thresholds={"mode": "absolute", "steps": [
+        {"color": "green", "value": None},
+        {"color": "yellow", "value": 24 * 14},
+        {"color": "red", "value": 24 * 45},
+    ]},
+    no_value="never run",
+    desc="Powered-on hours since the least recently tested drive last completed an "
+         "extended (long) self-test. Shows 'never run' when no drive has ever "
+         "completed one — which is the default state, since smartd schedules no tests "
+         "unless its config carries an -s directive.",
+))
+
+def _keyed(expr):
+    """Reduce a series to just the synthetic join key.
+
+    joinByField takes a single field, but a row here is identified by device *and*
+    test_type together. Joining on either one alone silently mismatches rows across
+    drives, so build one composite label and join on that instead.
+    """
+    return (f'max by (drive_test) (label_join({expr}, "drive_test", "/", '
+            f'"device", "test_type"))')
+
+
+panels.append(table(
+    "Self-Test History", [
+        # Only this query keeps device and test_type as their own columns.
+        ('label_join(smart_selftest_last_passed, "drive_test", "/", "device", '
+         '"test_type")', None),
+        (_keyed('smart_selftest_last_lifetime_hours'), None),
+        (_keyed('smart_selftest_count'), None),
+        # Powered-on hours elapsed since that test ran. group_right() keeps the
+        # right-hand side's labels, so device and test_type survive the subtraction.
+        (_keyed(power_on() + ' / 3600 - on(device) group_right() '
+                'smart_selftest_last_lifetime_hours'), None),
+    ],
+    gp(12, 54, 12, 10), keep="drive_test|device|test_type|Value.*",
+    renames={
+        "__join__": "drive_test", "device": "Device", "test_type": "Test",
+        "Value #A": "Last Result", "Value #B": "At Hours",
+        "Value #C": "Logged", "Value #D": "Hours Ago",
+    },
+    exclude=["drive_test"],
+    order=["device", "test_type", "Value #A", "Value #B", "Value #C", "Value #D"],
+    sort_by="Device", sort_desc=False,
+    overrides=[
+        {"matcher": {"id": "byName", "options": "Last Result"},
+         "properties": [
+             {"id": "mappings", "value": PASS_MAP},
+             {"id": "custom.cellOptions",
+              "value": {"type": "color-background", "mode": "basic"}},
+         ]},
+    ],
+    desc="Short and extended self-test results per drive, from "
+         "collectors/smart-selftest.py. 'At Hours' is the drive's powered-on hour "
+         "count when the test ran, straight from the log. 'Hours Ago' subtracts it "
+         "from the drive's current powered-on hours; on SAS that figure already falls "
+         "back to this log, so 0 means the newest log entry is also the highest hour "
+         "count the drive has ever reported — not that the test ran just now.",
+))
+
+# ======================================================================================
+# Collapsed: trends and diagnostics
+# ======================================================================================
+panels.append(row("Attribute Trends", 64, collapsed=True, children=[
+    timeseries(
+        "Reallocated Sectors (all SATA drives)",
+        [('smartctl_device_attribute{attribute_id="5", attribute_value_type="raw"}',
+          "{{device}}")],
+        gp(0, 65, 8, 7), minval=0,
+        custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+    ),
+    timeseries(
+        "Pending & Uncorrectable Sectors",
+        [('smartctl_device_attribute{attribute_id=~"197|198", '
+          'attribute_value_type="raw"}', "{{device}} {{attribute_name}}")],
+        gp(8, 65, 8, 7), minval=0,
+        custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+    ),
+    timeseries(
+        "CRC Errors (cabling)",
+        [('smartctl_device_attribute{attribute_id="199", attribute_value_type="raw"}',
+          "{{device}}")],
+        gp(16, 65, 8, 7), minval=0,
+        custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
+        desc="Attribute 199 counts errors on the SATA link rather than the platters. "
+             "Growth here usually means a cable or backplane, not a dying disk.",
+    ),
+    timeseries(
+        "Data Written / Read (SAS)", [
+            ('smartctl_device_bytes_written', "{{device}} written"),
+            ('smartctl_device_bytes_read', "{{device}} read"),
+        ],
+        gp(0, 72, 12, 7), unit="bytes", minval=0,
+    ),
+    timeseries(
+        "Health Code History",
+        [('smartctl:health_code', "{{device}}")],
+        gp(12, 72, 12, 7), mappings=HEALTH_MAP, minval=0, maxval=2,
+        custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"},
+    ),
+]))
+
+panels.append(row("Collector Health", 65, collapsed=True, children=[
+    timeseries(
+        "smartctl Exit Status",
+        [('smartctl_device_smartctl_exit_status', "{{device}}")],
+        gp(0, 66, 8, 7), minval=0,
+        desc="Non-zero means smartctl itself had trouble reading the device. Bit 0 is "
+             "a command-line problem, bit 1 a device-open failure, bit 3 a failing "
+             "drive.",
+    ),
+    timeseries(
+        "Self-Test Collector Duration",
+        [('smart_selftest_collection_duration_seconds', "collection")],
+        gp(8, 66, 8, 7), unit="s", minval=0,
+    ),
+    timeseries(
+        "Devices Seen", [
+            ('smartctl_devices', "exporter"),
+            ('smart_selftest_devices', "self-test collector"),
+        ],
+        gp(16, 66, 8, 7), minval=0,
+        desc="These should agree. A divergence means one of the two cannot see some "
+             "drives.",
+    ),
+]))
+
+print(json.dumps(dashboard(
+    title="Drive Health (SMART)",
+    uid="drive-health-smart",
+    description="Per-drive SMART health in the style of CrystalDiskInfo: a "
+                "Good/Caution/Bad verdict, full raw attribute tables for SATA, SCSI "
+                "error counters for SAS, and self-test history.",
+    panels=panels,
+    variables=[
+        # smartctl:row is one rendered line per drive, "sde · GOOD · 6.0 TB · ...".
+        # The nested capture groups keep the *value* a bare device name, so every
+        # `device=~"$ata_device"` selector below is unaffected, while the *text* the UI
+        # shows -- and that ${ata_device:text} puts in the row title -- is the whole
+        # line. That is what makes a collapsed row identifiable without opening it.
+        query_var("ata_device", 'label_values(smartctl:row{bus="ata"}, row)',
+                  label="SATA drive", regex=ROW_REGEX),
+        query_var("sas_device", 'label_values(smartctl:row{bus="sas"}, row)',
+                  label="SAS drive", regex=ROW_REGEX),
+    ],
+    tags=["smart", "storage", "hardware"],
+), indent=2))