path: root/dashboards/gen_smart.py

#!/usr/bin/env python3
"""Generate the "Drive Health (SMART)" Grafana dashboard.

modules/dashboards.nix runs this at build time; the JSON is not committed. To see what
it produces:

    python3 dashboards/gen_smart.py | jq .

A CrystalDiskInfo-style view: a Good/Caution/Bad verdict per drive, the full raw SMART
attribute table, and self-test history.

The fleet is mixed, so there are two repeating sections:

* SATA drives expose the classic ATA attribute table (value / worst / thresh / raw).
* SAS drives expose none of it. Their health lives in SCSI error counters instead --
  grown defect list, uncorrected errors, corrected-error breakdown.

Drives are routed into the right section by *metric presence* rather than by the
`protocol` label, which is more robust: only ATA drives have smartctl_device_attribute,
and only SCSI drives have smartctl_scsi_grown_defect_list.

Metrics used:

* smartctl_*                 -- prometheus-community/smartctl_exporter
* smartctl:health_code etc.  -- recording rules in rules/smartctl.yml
* smart_selftest_*           -- collectors/smart-selftest.py
"""

import json

from panels import (OK_BAD, dashboard, gp, query_var, row, stat, table,
                    timeseries)

# Matches the verdict encoding in rules/smartctl.yml.
HEALTH_MAP = [{
    "type": "value",
    "options": {
        "0": {"text": "GOOD", "color": "green", "index": 0},
        "1": {"text": "CAUTION", "color": "orange", "index": 1},
        "2": {"text": "BAD", "color": "red", "index": 2},
    },
}]

HEALTH_THRESHOLDS = {"mode": "absolute", "steps": [
    {"color": "green", "value": None},
    {"color": "orange", "value": 1},
    {"color": "red", "value": 2},
]}

PASS_MAP = [{
    "type": "value",
    "options": {
        "0": {"text": "FAILED", "color": "red", "index": 0},
        "1": {"text": "passed", "color": "green", "index": 1},
    },
}]

# Every query on this dashboard is scoped to exactly one machine. The host variable is
# deliberately single-select with no "All": `sda` is not a unique name across a fleet,
# and the `max by (device)` reductions and joinByField transformations below would
# silently merge two different physical drives into one row if more than one host were
# ever selected at a time.
HOST = 'host=~"$host"'
ATA = f'{HOST}, device=~"$ata_device"'
SAS = f'{HOST}, device=~"$sas_device"'

# The per-drive sections come from smartctl:row, one preformatted line per drive.
#
# It has to be query_result() rather than the more obvious
# label_values(smartctl:row{...}, row), because label_values() asks Prometheus for every
# value of `row` seen anywhere in the dashboard's time range -- 24 hours by default.
# Replace a drive and that window still holds the line the *old* drive produced. Both
# lines then reduce to the same variable value (the device name, which the panels below
# select on), and Grafana's uniqBy(options, 'value') keeps whichever came back first.
# The result is one row headed with the removed drive's model and serial, no row for its
# replacement, and panels underneath showing the new drive's numbers. query_result() is
# an instant query at the end of the range, so it can only return drives that are in the
# machine now.
DEVICE_QUERY = 'query_result(smartctl:row{{bus="{bus}", host=~"$host"}})'

# Pulls the value out of a query_result() line, which Grafana renders as
#   smartctl:row{bus="sas",device="sde",...,row="sde · GOOD · 6.0 TB · ...",...} 0 17883…
# Anchoring on `,row="` rather than `row="` keeps it from matching a `row` that turned up
# inside some other label's text. The groups nest rather than sit side by side, which is
# what lets `text` (the whole line, shown in the row title) be a superset of `value` (the
# bare device name, which every device=~"$..." selector below substitutes).
ROW_REGEX = r'/,row="(?<text>(?<value>[^\s"]+)[^"]*)"/'


def power_on(sel=""):
    """Best available powered-on seconds per drive, in seconds.

    On ATA, smartctl reads power_on_time straight from attribute 9 and it is reliable.
    On SAS there is no such attribute: smartctl sources power_on_time from the
    Background Scan Results log page, whose "Accumulated power on time" counter is
    maintained by the background-scan subsystem rather than being the drive's lifetime
    meter. On the HGST SAS drives here it reads a few thousand hours while the drives'
    own self-test logs record tens of thousands.

    The self-test log's lifetime-hours column is stamped by the drive at the moment
    each test ran, so it can never exceed the true powered-on figure. Taking whichever
    source is larger is therefore never an over-estimate, and recovers a sane number on
    drives whose power-on counter under-counts or was reset.
    """
    braces = "{" + ", ".join(x for x in (HOST, sel) if x) + "}"
    return (f'max by (device) (smartctl_device_power_on_seconds{braces} '
            f'or (smart_selftest_last_lifetime_hours{braces} * 3600))')


def drive_info(title, sel, pos, label_cols, renames, desc=""):
    """One-row identity card for a drive: what it is, not how it is doing.

    smartctl_device is an info metric whose value is always 1 and whose payload is
    entirely in its labels, so the value column gets dropped. Rotation rate and
    capacity are joined in as real numbers; the join is an outer one so an SSD, which
    reports no rotation rate at all, still shows up with a blank RPM cell.
    """
    return table(
        title, [
            (f'smartctl_device{{{sel}}}', None),
            (f'max by (device) (smartctl_device_rotation_rate{{{sel}}})', None),
            (f'max by (device) (smartctl_device_capacity_bytes{{{sel}}})', None),
        ],
        pos,
        keep="device|" + "|".join(label_cols) + "|Value.*",
        renames={"__join__": "device", "device": "Device",
                 "Value #B": "RPM", "Value #C": "Capacity", **renames},
        exclude=["Value #A"],
        order=["device", *label_cols, "Value #B", "Value #C"],
        overrides=[
            {"matcher": {"id": "byName", "options": "Capacity"},
             "properties": [{"id": "unit", "value": "bytes"}]},
            {"matcher": {"id": "byName", "options": "RPM"},
             "properties": [{"id": "unit", "value": "rotrpm"},
                            {"id": "noValue", "value": "SSD"}]},
        ],
        desc=desc,
    )


panels = []

# ======================================================================================
# Fleet Overview
# ======================================================================================
panels.append(row("Fleet Overview", 0))

panels.append(stat(
    "Healthy", [(f'count(smartctl:health_code{{{HOST}}} == 0) or vector(0)', "drives")],
    gp(0, 1, 4, 4), colormode="background",
    thresholds={"mode": "absolute", "steps": [{"color": "green", "value": None}]},
    desc="Drives with no failing attribute, no SCSI uncorrected errors and a passing "
         "SMART self-assessment.",
))
panels.append(stat(
    "Caution", [(f'count(smartctl:health_code{{{HOST}}} == 1) or vector(0)', "drives")],
    gp(4, 1, 4, 4), colormode="background",
    thresholds={"mode": "absolute", "steps": [
        {"color": "green", "value": None}, {"color": "orange", "value": 1}]},
    desc="Reallocated / pending / uncorrectable sectors on SATA, or a non-empty grown "
         "defect list on SAS. Not failing yet — worth watching the trend.",
))
panels.append(stat(
    "Failing", [(f'count(smartctl:health_code{{{HOST}}} == 2) or vector(0)', "drives")],
    gp(8, 1, 4, 4), colormode="background", thresholds=OK_BAD,
    desc="SMART self-assessment failed, an attribute is at or below its threshold, or "
         "the drive reported errors it could not correct. Replace these.",
))
panels.append(stat(
    "Drives", [(f'count(smartctl_device_smart_status{{{HOST}}})', "total")],
    gp(12, 1, 4, 4),
))
panels.append(stat(
    "Hottest Drive",
    [(f'max(smartctl_device_temperature{{{HOST}, temperature_type="current"}})',
      "max")],
    gp(16, 1, 4, 4), unit="celsius", colormode="background",
    thresholds={"mode": "absolute", "steps": [
        {"color": "green", "value": None},
        {"color": "yellow", "value": 45},
        {"color": "red", "value": 55},
    ]},
))
panels.append(stat(
    "Longest Powered On", [(f'max({power_on()})', "max")],
    gp(20, 1, 4, 4), unit="s",
))

panels.append(table(
    "All Drives", [
        # group_left carries the descriptive labels from the info metric onto the
        # verdict, so this one query supplies device, model, serial and protocol.
        (f'smartctl:health_code{{{HOST}}} * on(device) group_left(model_name, '
         f'serial_number, protocol) smartctl_device{{{HOST}}}', None),
        (f'max by (device) (smartctl_device_temperature{{{HOST}, '
         f'temperature_type="current"}})', None),
        (power_on(), None),
        (f'max by (device) (smartctl_device_capacity_bytes{{{HOST}}})', None),
    ],
    gp(0, 5, 16, 10), keep="device|model_name|serial_number|protocol|Value.*",
    renames={
        "__join__": "device", "device": "Device", "model_name": "Model",
        "serial_number": "Serial", "protocol": "Bus",
        "Value #A": "Health", "Value #B": "Temp",
        "Value #C": "Powered On", "Value #D": "Capacity",
    },
    order=["device", "model_name", "serial_number", "protocol",
           "Value #A", "Value #B", "Value #C", "Value #D"],
    sort_by="Health",
    overrides=[
        {"matcher": {"id": "byName", "options": "Health"},
         "properties": [
             {"id": "mappings", "value": HEALTH_MAP},
             {"id": "thresholds", "value": HEALTH_THRESHOLDS},
             {"id": "custom.cellOptions",
              "value": {"type": "color-background", "mode": "basic"}},
         ]},
        {"matcher": {"id": "byName", "options": "Temp"},
         "properties": [{"id": "unit", "value": "celsius"}]},
        {"matcher": {"id": "byName", "options": "Powered On"},
         "properties": [{"id": "unit", "value": "s"}]},
        {"matcher": {"id": "byName", "options": "Capacity"},
         "properties": [{"id": "unit", "value": "bytes"}]},
    ],
    desc="Every drive, worst first. Health is computed by rules/smartctl.yml.",
))

panels.append(timeseries(
    "Temperatures",
    [(f'smartctl_device_temperature{{{HOST}, temperature_type="current"}}',
      "{{device}}")],
    gp(16, 5, 8, 10), unit="celsius",
    desc="Sustained operation above ~45C measurably shortens drive life.",
))

# ======================================================================================
# SATA — repeats once per ATA drive
# ======================================================================================
panels.append(row("${ata_device:text}", 15, repeat="ata_device"))

panels.append(stat(
    "Health", [(f'smartctl:health_code{{{ATA}}}', "{{device}}")],
    gp(0, 16, 6, 4), mappings=HEALTH_MAP, thresholds=HEALTH_THRESHOLDS,
    colormode="background",
))
panels.append(stat(
    "Temperature",
    [(f'smartctl_device_temperature{{{ATA}, temperature_type="current"}}', "temp")],
    gp(6, 16, 3, 4), unit="celsius",
    thresholds={"mode": "absolute", "steps": [
        {"color": "green", "value": None},
        {"color": "yellow", "value": 45},
        {"color": "red", "value": 55},
    ]},
))
panels.append(stat(
    "Powered On", [(power_on(ATA), "hours")],
    gp(9, 16, 3, 4), unit="s",
))
panels.append(stat(
    "Capacity", [(f'smartctl_device_capacity_bytes{{{ATA}}}', "bytes")],
    gp(12, 16, 3, 4), unit="bytes",
))
panels.append(stat(
    "Power Cycles", [(f'smartctl_device_power_cycle_count{{{ATA}}}', "cycles")],
    gp(15, 16, 3, 4),
))
panels.append(stat(
    "Reallocated Sectors",
    [(f'smartctl_device_attribute{{{ATA}, attribute_id="5", '
      f'attribute_value_type="raw"}}', "sectors")],
    gp(18, 16, 3, 4), thresholds=OK_BAD, colormode="background",
    desc="Attribute 5 raw. Sectors the drive has already retired. Any growth here is "
         "the single most predictive sign of impending failure.",
))
panels.append(stat(
    "Pending Sectors",
    [(f'smartctl_device_attribute{{{ATA}, attribute_id="197", '
      f'attribute_value_type="raw"}}', "sectors")],
    gp(21, 16, 3, 4), thresholds=OK_BAD, colormode="background",
    desc="Attribute 197 raw. Sectors that failed to read and are waiting to be "
         "reallocated on next write. Often precedes reallocations.",
))

panels.append(drive_info(
    "Drive — $ata_device", ATA, gp(0, 20, 24, 4),
    label_cols=["model_family", "model_name", "serial_number", "firmware_version",
                "interface", "sata_version", "form_factor"],
    renames={"model_family": "Family", "model_name": "Model",
             "serial_number": "Serial", "firmware_version": "Firmware",
             "interface": "Interface", "sata_version": "SATA", "form_factor": "Form"},
    desc="Identity of this drive. RPM shows SSD when the drive reports no rotation "
         "rate.",
))

panels.append(table(
    "SMART Attributes — $ata_device", [
        # Only this query keeps the descriptive labels; the rest are reduced to the
        # join key so nothing collides in joinByField.
        (f'smartctl_device_attribute{{{ATA}, attribute_value_type="value"}}', None),
        (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
         f'attribute_value_type="worst"}})', None),
        (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
         f'attribute_value_type="thresh"}})', None),
        (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
         f'attribute_value_type="raw"}})', None),
        # How much headroom is left before this attribute trips its own failure
        # threshold. Zero or below is what SMART calls a failure.
        (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, '
         f'attribute_value_type="value"}} - on(device, attribute_id) '
         f'smartctl_device_attribute{{{ATA}, attribute_value_type="thresh"}})', None),
    ],
    gp(0, 24, 24, 12), keep="attribute_id|attribute_name|Value.*",
    renames={
        "__join__": "attribute_id", "attribute_id": "ID",
        "attribute_name": "Attribute",
        "Value #A": "Current", "Value #B": "Worst", "Value #C": "Threshold",
        "Value #D": "Raw", "Value #E": "Margin",
    },
    order=["attribute_id", "attribute_name",
           "Value #A", "Value #B", "Value #C", "Value #D", "Value #E"],
    sort_by="Attribute", sort_desc=False,
    overrides=[
        # Margin is the honest place to highlight trouble: a blanket threshold on Raw
        # would flag Power_On_Hours, which is enormous and perfectly healthy.
        {"matcher": {"id": "byName", "options": "Margin"},
         "properties": [
             {"id": "thresholds", "value": {"mode": "absolute", "steps": [
                 {"color": "red", "value": None},
                 {"color": "orange", "value": 1},
                 {"color": "yellow", "value": 10},
                 {"color": "green", "value": 25},
             ]}},
             {"id": "custom.cellOptions",
              "value": {"type": "color-background", "mode": "basic"}},
         ]},
    ],
    desc="Every attribute the drive reports, with its raw value. Current/Worst are "
         "normalised 0-253 scores (higher is better); Threshold is where the vendor "
         "declares failure; Margin is Current minus Threshold. ID sorts as text, so "
         "the table is ordered by attribute name.",
))

# ======================================================================================
# SAS — repeats once per SCSI drive
# ======================================================================================
panels.append(row("${sas_device:text}", 36, repeat="sas_device"))

panels.append(stat(
    "Health", [(f'smartctl:health_code{{{SAS}}}', "{{device}}")],
    gp(0, 37, 6, 4), mappings=HEALTH_MAP, thresholds=HEALTH_THRESHOLDS,
    colormode="background",
))
panels.append(stat(
    "Temperature",
    [(f'smartctl_device_temperature{{{SAS}, temperature_type="current"}}', "temp")],
    gp(6, 37, 3, 4), unit="celsius",
    thresholds={"mode": "absolute", "steps": [
        {"color": "green", "value": None},
        {"color": "yellow", "value": 45},
        {"color": "red", "value": 55},
    ]},
))
panels.append(stat(
    "Powered On", [(power_on(SAS), "hours")],
    gp(9, 37, 3, 4), unit="s",
    desc="SAS drives have no ATA attribute 9, so smartctl reports power_on_time from "
         "the Background Scan Results log page — a counter the background-scan "
         "subsystem maintains, not the drive's lifetime meter, and one that commonly "
         "reads far too low. This tile takes the larger of that counter and the "
         "lifetime-hours stamp in the drive's own self-test log, which is why it can "
         "exceed what `smartctl -x` prints as 'Accumulated power on time'.",
))
panels.append(stat(
    "Capacity", [(f'smartctl_device_capacity_bytes{{{SAS}}}', "bytes")],
    gp(12, 37, 3, 4), unit="bytes",
))
panels.append(stat(
    "Link Speed",
    [(f'smartctl_device_interface_speed{{{SAS}, speed_type="current"}} * 1000', "bps")],
    gp(15, 37, 3, 4), unit="bps",
    desc="Negotiated SAS link rate. A drive that has fallen back to a lower rate is "
         "worth investigating — usually a cable or expander.",
))
panels.append(stat(
    "Grown Defects", [(f'smartctl_scsi_grown_defect_list{{{SAS}}}', "defects")],
    gp(18, 37, 3, 4), thresholds=OK_BAD, colormode="background",
    desc="Sectors reallocated since the drive left the factory — the SAS equivalent "
         "of ATA attribute 5. A stable non-zero count is tolerable; growth is not.",
))
panels.append(stat(
    "Uncorrected Errors",
    [(f'smartctl_read_total_uncorrected_errors{{{SAS}}} + '
      f'smartctl_write_total_uncorrected_errors{{{SAS}}}', "errors")],
    gp(21, 37, 3, 4), thresholds=OK_BAD, colormode="background",
    desc="Errors the drive could not correct with ECC or retries, which means data "
         "was lost. Should be zero.",
))

panels.append(drive_info(
    "Drive — $sas_device", SAS, gp(0, 41, 24, 4),
    label_cols=["scsi_vendor", "model_name", "serial_number", "scsi_revision",
                "scsi_version", "protocol", "form_factor"],
    renames={"scsi_vendor": "Vendor", "model_name": "Model",
             "serial_number": "Serial", "scsi_revision": "Revision",
             "scsi_version": "SCSI", "protocol": "Bus", "form_factor": "Form"},
    desc="Identity of this drive. SAS drives leave firmware_version empty and report "
         "their revision in scsi_revision instead.",
))

panels.append(timeseries(
    "Corrected Errors — $sas_device", [
        (f'smartctl_read_errors_corrected_by_eccfast{{{SAS}}}', "read ECC fast"),
        (f'smartctl_read_errors_corrected_by_eccdelayed{{{SAS}}}', "read ECC delayed"),
        (f'smartctl_read_errors_corrected_by_rereads_rewrites{{{SAS}}}', "read retries"),
        (f'smartctl_write_errors_corrected_by_eccfast{{{SAS}}}', "write ECC fast"),
        (f'smartctl_write_errors_corrected_by_eccdelayed{{{SAS}}}',
         "write ECC delayed"),
        (f'smartctl_write_errors_corrected_by_rereads_rewrites{{{SAS}}}',
         "write retries"),
    ],
    gp(0, 45, 12, 8), minval=0,
    desc="Errors the drive fixed on its own. ECC-fast corrections are routine and "
         "grow constantly. Delayed corrections and especially rewrites/retries "
         "climbing is the signal worth watching.",
))
panels.append(timeseries(
    "Defects & Uncorrected — $sas_device", [
        (f'smartctl_scsi_grown_defect_list{{{SAS}}}', "grown defects"),
        (f'smartctl_read_total_uncorrected_errors{{{SAS}}}', "read uncorrected"),
        (f'smartctl_write_total_uncorrected_errors{{{SAS}}}', "write uncorrected"),
    ],
    gp(12, 45, 12, 8), minval=0,
    custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
    desc="Flat lines are good. Any upward step here is a real event.",
))

# ======================================================================================
# Self-tests
# ======================================================================================
panels.append(row("Self-Tests", 53))

panels.append(stat(
    "Drives Never Tested",
    [(f'count(smartctl_device_smart_status{{{HOST}}}) - '
      f'count(count by (device) (smart_selftest_last_lifetime_hours{{{HOST}}})) or '
      f'count(smartctl_device_smart_status{{{HOST}}})', "drives")],
    gp(0, 54, 4, 5), thresholds=OK_BAD, colormode="background",
    desc="Drives with no self-test log entries at all. smartd only runs tests if its "
         "config includes an -s schedule; merely enabling the service does not.",
))
panels.append(stat(
    "Failed Self-Tests",
    [(f'count(smart_selftest_last_passed{{{HOST}}} == 0) or vector(0)', "tests")],
    gp(4, 54, 4, 5), thresholds=OK_BAD, colormode="background",
))
panels.append(stat(
    "Oldest Long Test",
    [(f'max({power_on()} / 3600 - on(device) group_right() '
      f'smart_selftest_last_lifetime_hours{{{HOST}, test_type="extended"}})',
      "hours ago")],
    gp(8, 54, 4, 5), unit="h",
    thresholds={"mode": "absolute", "steps": [
        {"color": "green", "value": None},
        {"color": "yellow", "value": 24 * 14},
        {"color": "red", "value": 24 * 45},
    ]},
    no_value="never run",
    desc="Powered-on hours since the least recently tested drive last completed an "
         "extended (long) self-test. Shows 'never run' when no drive has ever "
         "completed one — which is the default state, since smartd schedules no tests "
         "unless its config carries an -s directive.",
))

def _keyed(expr):
    """Reduce a series to just the synthetic join key.

    joinByField takes a single field, but a row here is identified by device *and*
    test_type together. Joining on either one alone silently mismatches rows across
    drives, so build one composite label and join on that instead.
    """
    return (f'max by (drive_test) (label_join({expr}, "drive_test", "/", '
            f'"device", "test_type"))')


panels.append(table(
    "Self-Test History", [
        # Only this query keeps device and test_type as their own columns.
        (f'label_join(smart_selftest_last_passed{{{HOST}}}, "drive_test", "/", '
         f'"device", "test_type")', None),
        (_keyed(f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None),
        (_keyed(f'smart_selftest_count{{{HOST}}}'), None),
        # Powered-on hours elapsed since that test ran. group_right() keeps the
        # right-hand side's labels, so device and test_type survive the subtraction.
        (_keyed(power_on() + ' / 3600 - on(device) group_right() '
                f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None),
    ],
    gp(12, 54, 12, 10), keep="drive_test|device|test_type|Value.*",
    renames={
        "__join__": "drive_test", "device": "Device", "test_type": "Test",
        "Value #A": "Last Result", "Value #B": "At Hours",
        "Value #C": "Logged", "Value #D": "Hours Ago",
    },
    exclude=["drive_test"],
    order=["device", "test_type", "Value #A", "Value #B", "Value #C", "Value #D"],
    sort_by="Device", sort_desc=False,
    overrides=[
        {"matcher": {"id": "byName", "options": "Last Result"},
         "properties": [
             {"id": "mappings", "value": PASS_MAP},
             {"id": "custom.cellOptions",
              "value": {"type": "color-background", "mode": "basic"}},
         ]},
    ],
    desc="Short and extended self-test results per drive, from "
         "collectors/smart-selftest.py. 'At Hours' is the drive's powered-on hour "
         "count when the test ran, straight from the log. 'Hours Ago' subtracts it "
         "from the drive's current powered-on hours; on SAS that figure already falls "
         "back to this log, so 0 means the newest log entry is also the highest hour "
         "count the drive has ever reported — not that the test ran just now.",
))

# ======================================================================================
# Collapsed: trends and diagnostics
# ======================================================================================
panels.append(row("Attribute Trends", 64, collapsed=True, children=[
    timeseries(
        "Reallocated Sectors (all SATA drives)",
        [(f'smartctl_device_attribute{{{HOST}, attribute_id="5", '
          f'attribute_value_type="raw"}}', "{{device}}")],
        gp(0, 65, 8, 7), minval=0,
        custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
    ),
    timeseries(
        "Pending & Uncorrectable Sectors",
        [(f'smartctl_device_attribute{{{HOST}, attribute_id=~"197|198", '
          f'attribute_value_type="raw"}}', "{{device}} {{attribute_name}}")],
        gp(8, 65, 8, 7), minval=0,
        custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
    ),
    timeseries(
        "CRC Errors (cabling)",
        [(f'smartctl_device_attribute{{{HOST}, attribute_id="199", '
          f'attribute_value_type="raw"}}', "{{device}}")],
        gp(16, 65, 8, 7), minval=0,
        custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
        desc="Attribute 199 counts errors on the SATA link rather than the platters. "
             "Growth here usually means a cable or backplane, not a dying disk.",
    ),
    timeseries(
        "Data Written / Read (SAS)", [
            (f'smartctl_device_bytes_written{{{HOST}}}', "{{device}} written"),
            (f'smartctl_device_bytes_read{{{HOST}}}', "{{device}} read"),
        ],
        gp(0, 72, 12, 7), unit="bytes", minval=0,
    ),
    timeseries(
        "Health Code History",
        [(f'smartctl:health_code{{{HOST}}}', "{{device}}")],
        gp(12, 72, 12, 7), mappings=HEALTH_MAP, minval=0, maxval=2,
        custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"},
    ),
]))

panels.append(row("Collector Health", 65, collapsed=True, children=[
    timeseries(
        "smartctl Exit Status",
        [(f'smartctl_device_smartctl_exit_status{{{HOST}}}', "{{device}}")],
        gp(0, 66, 8, 7), minval=0,
        desc="Non-zero means smartctl itself had trouble reading the device. Bit 0 is "
             "a command-line problem, bit 1 a device-open failure, bit 3 a failing "
             "drive.",
    ),
    timeseries(
        "Self-Test Collector Duration",
        [(f'smart_selftest_collection_duration_seconds{{{HOST}}}', "collection")],
        gp(8, 66, 8, 7), unit="s", minval=0,
    ),
    timeseries(
        "Devices Seen", [
            (f'smartctl_devices{{{HOST}}}', "exporter"),
            (f'smart_selftest_devices{{{HOST}}}', "self-test collector"),
        ],
        gp(16, 66, 8, 7), minval=0,
        desc="These should agree. A divergence means one of the two cannot see some "
             "drives.",
    ),
]))

print(json.dumps(dashboard(
    title="Drive Health (SMART)",
    uid="drive-health-smart",
    description="Per-drive SMART health in the style of CrystalDiskInfo: a "
                "Good/Caution/Bad verdict, full raw attribute tables for SATA, SCSI "
                "error counters for SAS, and self-test history.",
    panels=panels,
    variables=[
        # One machine at a time. Not multi-select and no "All" -- see the comment on
        # HOST above for why selecting two hosts at once would silently merge same-named
        # drives rather than showing both. This one stays on label_values deliberately:
        # its window being the dashboard's time range is what keeps a machine that is
        # currently down still selectable, which is exactly when you want to look at it.
        query_var("host", "label_values(smartctl_device_smart_status, host)",
                  label="Host", multi=False, include_all=False),
        # smartctl:row is one rendered line per drive, "sde · GOOD · 6.0 TB · ...", and
        # ${ata_device:text} puts that whole line in the repeated row's title. That is
        # what makes a collapsed row identifiable without opening it. See DEVICE_QUERY
        # and ROW_REGEX above for why these are instant queries.
        query_var("ata_device", DEVICE_QUERY.format(bus="ata"),
                  label="SATA drive", regex=ROW_REGEX),
        query_var("sas_device", DEVICE_QUERY.format(bus="sas"),
                  label="SAS drive", regex=ROW_REGEX),
    ],
    tags=["smart", "storage", "hardware"],
), indent=2))