From a7f38335e5fed5a98bfaad64ab40544124337224 Mon Sep 17 00:00:00 2001 From: Tim Keller Date: Thu, 27 Aug 2026 10:13:13 -0500 Subject: initial commit --- rules/smartctl.yml | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 rules/smartctl.yml (limited to 'rules/smartctl.yml') diff --git a/rules/smartctl.yml b/rules/smartctl.yml new file mode 100644 index 0000000..dafed8d --- /dev/null +++ b/rules/smartctl.yml @@ -0,0 +1,116 @@ +# Drive health verdict, in the spirit of CrystalDiskInfo's Good / Caution / Bad. +# +# This lives in recording rules rather than in dashboard panels so the logic is written +# once and is reviewable on its own. The panels then just read smartctl:health_code. +# +# The fleet is mixed: SATA drives expose the classic ATA attribute table, SAS drives +# expose none of it and report health through SCSI error counters instead. Every rule +# below is therefore made *total* -- it ends in `or (count by (device) (...) * 0)` so +# that each series exists with value 0 for every drive, including drives of the other +# kind. Without that, the arithmetic in smartctl:health_code would silently drop drives +# whose label sets did not match on both sides. + +groups: + - name: smartctl + interval: 1m + rules: + # --- ATA ------------------------------------------------------------------ + # An attribute is failing when its normalised current value has fallen to or + # below the manufacturer's threshold. Thresholds of 0 mean "informational, never + # fails", so those are excluded rather than treated as an instant failure. + - record: smartctl:attr_failing:count + expr: | + count by (device) ( + smartctl_device_attribute{attribute_value_type="value"} + <= on(device, attribute_id) + (smartctl_device_attribute{attribute_value_type="thresh"} > 0) + ) + or (count by (device) (smartctl_device_smart_status) * 0) + + # Attributes whose raw value is a genuine warning sign across vendors: + # 5 Reallocated_Sector_Ct 187 Reported_Uncorrect + # 188 Command_Timeout 197 Current_Pending_Sector + # 198 Offline_Uncorrectable 199 UDMA_CRC_Error_Count (cabling) + # Deliberately narrow: attributes like 1 (Raw_Read_Error_Rate) carry huge raw + # values on Seagate drives by design and would flag healthy disks. + - record: smartctl:attr_caution:count + expr: | + count by (device) ( + smartctl_device_attribute{ + attribute_value_type="raw", + attribute_id=~"5|187|188|197|198|199" + } > 0 + ) + or (count by (device) (smartctl_device_smart_status) * 0) + + # --- SAS / SCSI ----------------------------------------------------------- + # Grown defect list: sectors reallocated since the drive left the factory. The + # SAS equivalent of attribute 5. Non-zero is worth watching, growth is worth + # acting on. + - record: smartctl:scsi_defects:count + expr: | + sum by (device) (smartctl_scsi_grown_defect_list) + or (count by (device) (smartctl_device_smart_status) * 0) + + # Errors the drive could not correct at all, which means data was lost. + - record: smartctl:scsi_uncorrected:count + expr: | + sum by (device) (smartctl_read_total_uncorrected_errors) + + sum by (device) (smartctl_write_total_uncorrected_errors) + or (count by (device) (smartctl_device_smart_status) * 0) + + # --- Verdict -------------------------------------------------------------- + # 0 good, 1 caution, 2 bad. The inner clamp_max on each group keeps two + # simultaneous caution signals from adding up to a false "bad". + - record: smartctl:health_code + expr: | + clamp_max( + 2 * clamp_max( + (max by (device) (smartctl_device_smart_status) == bool 0) + + (smartctl:attr_failing:count > bool 0) + + (smartctl:scsi_uncorrected:count > bool 0) + , 1) + + clamp_max( + (smartctl:attr_caution:count > bool 0) + + (smartctl:scsi_defects:count > bool 0) + , 1) + , 2) + + # --- Row labels ----------------------------------------------------------- + # Which section of the SMART dashboard a drive belongs in. Keyed on metric + # presence rather than the `protocol` label: only ATA drives report an attribute + # table, and only SCSI drives report a grown defect list. + - record: smartctl:bus + expr: | + label_replace(count by (device) (smartctl_device_attribute), "bus", "ata", "", "") + or + label_replace(count by (device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "") + + # The identity string from collectors/smart-selftest.py, made total so a drive + # still gets a row if that collector has not written its file yet. `unless` (not + # `or`) is what keeps the two halves disjoint -- both sides carry a `label`, so + # `or` would emit two series per drive rather than deduplicating. + - record: smartctl:ident + expr: | + max by (device, label) (smart_drive_label) + or + label_replace( + count by (device) (smartctl_device_smart_status) + unless on(device) smart_drive_label, + "label", "identity pending", "", "") + + # One preformatted line per drive: "sde · GOOD · 6.0 TB · SAS 7.2K · model · SN". + # Grafana can interpolate a variable's *display text* into a repeated row title + # (${sas_device:text}), which is the only way to label a collapsed row with live + # data -- so the whole line has to exist as a single label here. + - record: smartctl:row + expr: | + label_join( + ( + label_replace(smartctl:health_code == 0, "verdict", "GOOD", "", "") + or label_replace(smartctl:health_code == 1, "verdict", "CAUTION", "", "") + or label_replace(smartctl:health_code == 2, "verdict", "BAD", "", "") + ) + * on(device) group_left(bus) smartctl:bus + * on(device) group_left(label) smartctl:ident, + "row", " · ", "device", "verdict", "label") -- cgit v1.3.1