# Drive health verdict, in the spirit of CrystalDiskInfo's Good / Caution / Bad. # # This lives in recording rules rather than in dashboard panels so the logic is written # once and is reviewable on its own. The panels then just read smartctl:health_code. # # The fleet is mixed: SATA drives expose the classic ATA attribute table, SAS drives # expose none of it and report health through SCSI error counters instead. Every rule # below is therefore made *total* -- it ends in `or (count by (host, device) (...) * 0)` so # that each series exists with value 0 for every drive, including drives of the other # kind. Without that, the arithmetic in smartctl:health_code would silently drop drives # whose label sets did not match on both sides. groups: - name: smartctl interval: 1m rules: # --- ATA ------------------------------------------------------------------ # An attribute is failing when its normalised current value has fallen to or # below the manufacturer's threshold. Thresholds of 0 mean "informational, never # fails", so those are excluded rather than treated as an instant failure. - record: smartctl:attr_failing:count expr: | count by (host, device) ( smartctl_device_attribute{attribute_value_type="value"} <= on(host, device, attribute_id) (smartctl_device_attribute{attribute_value_type="thresh"} > 0) ) or (count by (host, device) (smartctl_device_smart_status) * 0) # Attributes whose raw value is a genuine warning sign across vendors: # 5 Reallocated_Sector_Ct 187 Reported_Uncorrect # 188 Command_Timeout 197 Current_Pending_Sector # 198 Offline_Uncorrectable 199 UDMA_CRC_Error_Count (cabling) # Deliberately narrow: attributes like 1 (Raw_Read_Error_Rate) carry huge raw # values on Seagate drives by design and would flag healthy disks. - record: smartctl:attr_caution:count expr: | count by (host, device) ( smartctl_device_attribute{ attribute_value_type="raw", attribute_id=~"5|187|188|197|198|199" } > 0 ) or (count by (host, device) (smartctl_device_smart_status) * 0) # --- SAS / SCSI ----------------------------------------------------------- # Grown defect list: sectors reallocated since the drive left the factory. The # SAS equivalent of attribute 5. Non-zero is worth watching, growth is worth # acting on. - record: smartctl:scsi_defects:count expr: | sum by (host, device) (smartctl_scsi_grown_defect_list) or (count by (host, device) (smartctl_device_smart_status) * 0) # Errors the drive could not correct at all, which means data was lost. - record: smartctl:scsi_uncorrected:count expr: | sum by (host, device) (smartctl_read_total_uncorrected_errors) + sum by (host, device) (smartctl_write_total_uncorrected_errors) or (count by (host, device) (smartctl_device_smart_status) * 0) # --- NVMe ----------------------------------------------------------------- # NVMe has neither an ATA attribute table nor SCSI error counters. Its health is # in the SMART/Health Information log, which smartctl_exporter emits only when the # interface is nvme. # # Each arm is reduced to {host, device} *before* the `or`, so the union # deduplicates per drive rather than emitting one series per metric name -- the # count is then "did anything fire", which is all smartctl:health_code asks. - record: smartctl:nvme_failing:count expr: | ( count by (host, device) (smartctl_device_critical_warning > 0) or count by (host, device) (smartctl_device_media_errors > 0) or count by (host, device) ( smartctl_device_available_spare < on(host, device) smartctl_device_available_spare_threshold ) ) or (count by (host, device) (smartctl_device_smart_status) * 0) # percentage_used is the controller's own estimate of consumed write endurance, # where 100 means the drive has reached its rated lifetime. Drives routinely run # well past it, so this is a warning and not a failure. - record: smartctl:nvme_caution:count expr: | count by (host, device) (smartctl_device_percentage_used >= 90) or (count by (host, device) (smartctl_device_smart_status) * 0) # --- Verdict -------------------------------------------------------------- # 0 good, 1 caution, 2 bad. The inner clamp_max on each group keeps two # simultaneous caution signals from adding up to a false "bad". - record: smartctl:health_code expr: | clamp_max( 2 * clamp_max( (max by (host, device) (smartctl_device_smart_status) == bool 0) + (smartctl:attr_failing:count > bool 0) + (smartctl:scsi_uncorrected:count > bool 0) + (smartctl:nvme_failing:count > bool 0) , 1) + clamp_max( (smartctl:attr_caution:count > bool 0) + (smartctl:scsi_defects:count > bool 0) + (smartctl:nvme_caution:count > bool 0) , 1) , 2) # --- Row labels ----------------------------------------------------------- # Which section of the SMART dashboard a drive belongs in. Keyed on metric # presence rather than the `protocol` label: only ATA drives report an attribute # table, only SCSI drives report a grown defect list, and smartctl_exporter emits # percentage_used only for nvme. The three are disjoint by construction, so the # order of the `or` arms does not matter -- but a drive matching none of them gets # no bus, drops out of the smartctl:row join, and silently loses its dashboard # section, which is what NVMe used to do. - record: smartctl:bus expr: | label_replace(count by (host, device) (smartctl_device_attribute), "bus", "ata", "", "") or label_replace(count by (host, device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "") or label_replace(count by (host, device) (smartctl_device_percentage_used), "bus", "nvme", "", "") # The identity string from collectors/smart-selftest.py, made total so a drive # still gets a row if that collector has not written its file yet. `unless` (not # `or`) is what keeps the two halves disjoint -- both sides carry a `label`, so # `or` would emit two series per drive rather than deduplicating. - record: smartctl:ident expr: | max by (host, device, label) (smart_drive_label) or label_replace( count by (host, device) (smartctl_device_smart_status) unless on(host, device) smart_drive_label, "label", "identity pending", "", "") # One preformatted line per drive: "sde · GOOD · 6.0 TB · SAS 7.2K · model · SN". # Grafana can interpolate a variable's *display text* into a repeated row title # (${sas_device:text}), which is the only way to label a collapsed row with live # data -- so the whole line has to exist as a single label here. - record: smartctl:row expr: | label_join( ( label_replace(smartctl:health_code == 0, "verdict", "GOOD", "", "") or label_replace(smartctl:health_code == 1, "verdict", "CAUTION", "", "") or label_replace(smartctl:health_code == 2, "verdict", "BAD", "", "") ) * on(host, device) group_left(bus) smartctl:bus * on(host, device) group_left(label) smartctl:ident, "row", " · ", "device", "verdict", "label")