1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
# Drive health verdict, in the spirit of CrystalDiskInfo's Good / Caution / Bad.
#
# This lives in recording rules rather than in dashboard panels so the logic is written
# once and is reviewable on its own. The panels then just read smartctl:health_code.
#
# The fleet is mixed: SATA drives expose the classic ATA attribute table, SAS drives
# expose none of it and report health through SCSI error counters instead. Every rule
# below is therefore made *total* -- it ends in `or (count by (host, device) (...) * 0)` so
# that each series exists with value 0 for every drive, including drives of the other
# kind. Without that, the arithmetic in smartctl:health_code would silently drop drives
# whose label sets did not match on both sides.
groups:
- name: smartctl
interval: 1m
rules:
# --- ATA ------------------------------------------------------------------
# An attribute is failing when its normalised current value has fallen to or
# below the manufacturer's threshold. Thresholds of 0 mean "informational, never
# fails", so those are excluded rather than treated as an instant failure.
- record: smartctl:attr_failing:count
expr: |
count by (host, device) (
smartctl_device_attribute{attribute_value_type="value"}
<= on(host, device, attribute_id)
(smartctl_device_attribute{attribute_value_type="thresh"} > 0)
)
or (count by (host, device) (smartctl_device_smart_status) * 0)
# Attributes whose raw value is a genuine warning sign across vendors:
# 5 Reallocated_Sector_Ct 187 Reported_Uncorrect
# 188 Command_Timeout 197 Current_Pending_Sector
# 198 Offline_Uncorrectable 199 UDMA_CRC_Error_Count (cabling)
# Deliberately narrow: attributes like 1 (Raw_Read_Error_Rate) carry huge raw
# values on Seagate drives by design and would flag healthy disks.
- record: smartctl:attr_caution:count
expr: |
count by (host, device) (
smartctl_device_attribute{
attribute_value_type="raw",
attribute_id=~"5|187|188|197|198|199"
} > 0
)
or (count by (host, device) (smartctl_device_smart_status) * 0)
# --- SAS / SCSI -----------------------------------------------------------
# Grown defect list: sectors reallocated since the drive left the factory. The
# SAS equivalent of attribute 5. Non-zero is worth watching, growth is worth
# acting on.
- record: smartctl:scsi_defects:count
expr: |
sum by (host, device) (smartctl_scsi_grown_defect_list)
or (count by (host, device) (smartctl_device_smart_status) * 0)
# Errors the drive could not correct at all, which means data was lost.
- record: smartctl:scsi_uncorrected:count
expr: |
sum by (host, device) (smartctl_read_total_uncorrected_errors)
+ sum by (host, device) (smartctl_write_total_uncorrected_errors)
or (count by (host, device) (smartctl_device_smart_status) * 0)
# --- NVMe -----------------------------------------------------------------
# NVMe has neither an ATA attribute table nor SCSI error counters. Its health is
# in the SMART/Health Information log, which smartctl_exporter emits only when the
# interface is nvme.
#
# Each arm is reduced to {host, device} *before* the `or`, so the union
# deduplicates per drive rather than emitting one series per metric name -- the
# count is then "did anything fire", which is all smartctl:health_code asks.
- record: smartctl:nvme_failing:count
expr: |
(
count by (host, device) (smartctl_device_critical_warning > 0)
or
count by (host, device) (smartctl_device_media_errors > 0)
or
count by (host, device) (
smartctl_device_available_spare
< on(host, device) smartctl_device_available_spare_threshold
)
)
or (count by (host, device) (smartctl_device_smart_status) * 0)
# percentage_used is the controller's own estimate of consumed write endurance,
# where 100 means the drive has reached its rated lifetime. Drives routinely run
# well past it, so this is a warning and not a failure.
- record: smartctl:nvme_caution:count
expr: |
count by (host, device) (smartctl_device_percentage_used >= 90)
or (count by (host, device) (smartctl_device_smart_status) * 0)
# --- Verdict --------------------------------------------------------------
# 0 good, 1 caution, 2 bad. The inner clamp_max on each group keeps two
# simultaneous caution signals from adding up to a false "bad".
- record: smartctl:health_code
expr: |
clamp_max(
2 * clamp_max(
(max by (host, device) (smartctl_device_smart_status) == bool 0)
+ (smartctl:attr_failing:count > bool 0)
+ (smartctl:scsi_uncorrected:count > bool 0)
+ (smartctl:nvme_failing:count > bool 0)
, 1)
+ clamp_max(
(smartctl:attr_caution:count > bool 0)
+ (smartctl:scsi_defects:count > bool 0)
+ (smartctl:nvme_caution:count > bool 0)
, 1)
, 2)
# --- Row labels -----------------------------------------------------------
# Which section of the SMART dashboard a drive belongs in. Keyed on metric
# presence rather than the `protocol` label: only ATA drives report an attribute
# table, only SCSI drives report a grown defect list, and smartctl_exporter emits
# percentage_used only for nvme. The three are disjoint by construction, so the
# order of the `or` arms does not matter -- but a drive matching none of them gets
# no bus, drops out of the smartctl:row join, and silently loses its dashboard
# section, which is what NVMe used to do.
- record: smartctl:bus
expr: |
label_replace(count by (host, device) (smartctl_device_attribute), "bus", "ata", "", "")
or
label_replace(count by (host, device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "")
or
label_replace(count by (host, device) (smartctl_device_percentage_used), "bus", "nvme", "", "")
# The identity string from collectors/smart-selftest.py, made total so a drive
# still gets a row if that collector has not written its file yet. `unless` (not
# `or`) is what keeps the two halves disjoint -- both sides carry a `label`, so
# `or` would emit two series per drive rather than deduplicating.
- record: smartctl:ident
expr: |
max by (host, device, label) (smart_drive_label)
or
label_replace(
count by (host, device) (smartctl_device_smart_status)
unless on(host, device) smart_drive_label,
"label", "identity pending", "", "")
# One preformatted line per drive: "sde · GOOD · 6.0 TB · SAS 7.2K · model · SN".
# Grafana can interpolate a variable's *display text* into a repeated row title
# (${sas_device:text}), which is the only way to label a collapsed row with live
# data -- so the whole line has to exist as a single label here.
- record: smartctl:row
expr: |
label_join(
(
label_replace(smartctl:health_code == 0, "verdict", "GOOD", "", "")
or label_replace(smartctl:health_code == 1, "verdict", "CAUTION", "", "")
or label_replace(smartctl:health_code == 2, "verdict", "BAD", "", "")
)
* on(host, device) group_left(bus) smartctl:bus
* on(host, device) group_left(label) smartctl:ident,
"row", " · ", "device", "verdict", "label")
|
