1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
# Drive health verdict, in the spirit of CrystalDiskInfo's Good / Caution / Bad.
#
# This lives in recording rules rather than in dashboard panels so the logic is written
# once and is reviewable on its own. The panels then just read smartctl:health_code.
#
# The fleet is mixed: SATA drives expose the classic ATA attribute table, SAS drives
# expose none of it and report health through SCSI error counters instead. Every rule
# below is therefore made *total* -- it ends in `or (count by (host, device) (...) * 0)` so
# that each series exists with value 0 for every drive, including drives of the other
# kind. Without that, the arithmetic in smartctl:health_code would silently drop drives
# whose label sets did not match on both sides.
groups:
- name: smartctl
interval: 1m
rules:
# --- ATA ------------------------------------------------------------------
# An attribute is failing when its normalised current value has fallen to or
# below the manufacturer's threshold. Thresholds of 0 mean "informational, never
# fails", so those are excluded rather than treated as an instant failure.
- record: smartctl:attr_failing:count
expr: |
count by (host, device) (
smartctl_device_attribute{attribute_value_type="value"}
<= on(host, device, attribute_id)
(smartctl_device_attribute{attribute_value_type="thresh"} > 0)
)
or (count by (host, device) (smartctl_device_smart_status) * 0)
# Attributes whose raw value is a genuine warning sign across vendors:
# 5 Reallocated_Sector_Ct 187 Reported_Uncorrect
# 188 Command_Timeout 197 Current_Pending_Sector
# 198 Offline_Uncorrectable 199 UDMA_CRC_Error_Count (cabling)
# Deliberately narrow: attributes like 1 (Raw_Read_Error_Rate) carry huge raw
# values on Seagate drives by design and would flag healthy disks.
- record: smartctl:attr_caution:count
expr: |
count by (host, device) (
smartctl_device_attribute{
attribute_value_type="raw",
attribute_id=~"5|187|188|197|198|199"
} > 0
)
or (count by (host, device) (smartctl_device_smart_status) * 0)
# --- SAS / SCSI -----------------------------------------------------------
# Grown defect list: sectors reallocated since the drive left the factory. The
# SAS equivalent of attribute 5. Non-zero is worth watching, growth is worth
# acting on.
- record: smartctl:scsi_defects:count
expr: |
sum by (host, device) (smartctl_scsi_grown_defect_list)
or (count by (host, device) (smartctl_device_smart_status) * 0)
# Errors the drive could not correct at all, which means data was lost.
- record: smartctl:scsi_uncorrected:count
expr: |
sum by (host, device) (smartctl_read_total_uncorrected_errors)
+ sum by (host, device) (smartctl_write_total_uncorrected_errors)
or (count by (host, device) (smartctl_device_smart_status) * 0)
# --- Verdict --------------------------------------------------------------
# 0 good, 1 caution, 2 bad. The inner clamp_max on each group keeps two
# simultaneous caution signals from adding up to a false "bad".
- record: smartctl:health_code
expr: |
clamp_max(
2 * clamp_max(
(max by (host, device) (smartctl_device_smart_status) == bool 0)
+ (smartctl:attr_failing:count > bool 0)
+ (smartctl:scsi_uncorrected:count > bool 0)
, 1)
+ clamp_max(
(smartctl:attr_caution:count > bool 0)
+ (smartctl:scsi_defects:count > bool 0)
, 1)
, 2)
# --- Row labels -----------------------------------------------------------
# Which section of the SMART dashboard a drive belongs in. Keyed on metric
# presence rather than the `protocol` label: only ATA drives report an attribute
# table, and only SCSI drives report a grown defect list.
- record: smartctl:bus
expr: |
label_replace(count by (host, device) (smartctl_device_attribute), "bus", "ata", "", "")
or
label_replace(count by (host, device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "")
# The identity string from collectors/smart-selftest.py, made total so a drive
# still gets a row if that collector has not written its file yet. `unless` (not
# `or`) is what keeps the two halves disjoint -- both sides carry a `label`, so
# `or` would emit two series per drive rather than deduplicating.
- record: smartctl:ident
expr: |
max by (host, device, label) (smart_drive_label)
or
label_replace(
count by (host, device) (smartctl_device_smart_status)
unless on(host, device) smart_drive_label,
"label", "identity pending", "", "")
# One preformatted line per drive: "sde · GOOD · 6.0 TB · SAS 7.2K · model · SN".
# Grafana can interpolate a variable's *display text* into a repeated row title
# (${sas_device:text}), which is the only way to label a collapsed row with live
# data -- so the whole line has to exist as a single label here.
- record: smartctl:row
expr: |
label_join(
(
label_replace(smartctl:health_code == 0, "verdict", "GOOD", "", "")
or label_replace(smartctl:health_code == 1, "verdict", "CAUTION", "", "")
or label_replace(smartctl:health_code == 2, "verdict", "BAD", "", "")
)
* on(host, device) group_left(bus) smartctl:bus
* on(host, device) group_left(label) smartctl:ident,
"row", " · ", "device", "verdict", "label")
|
