1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
# Alerting rules for the collectors' metrics.
#
# Loaded when services.telemetry.server.alerts.enable is on. Most of the conditions are
# one-liners because rules/smartctl.yml has already reduced drive health to a single
# smartctl:health_code per drive -- see that file for how the verdict is derived.
#
# Every alert carries the `host` label, so notifications say which machine. That label
# comes from the relabelling in modules/server.nix, not from the exporters.
#
# These are deliberately conservative: long `for` durations, and thresholds chosen so a
# healthy fleet is silent. If you want different numbers, set
# services.telemetry.server.alerts.defaultRules = false and point
# services.telemetry.server.prometheus.ruleFiles at your own copy.
groups:
- name: drives
rules:
# smartctl:health_code is 2 when the drive's own SMART self-assessment failed, an
# attribute is at or below its vendor threshold, or the drive reported errors it
# could not correct. Any of those means replace it.
- alert: DriveFailing
expr: smartctl:health_code == 2
for: 15m
labels:
severity: critical
annotations:
summary: "{{ $labels.host }}: drive {{ $labels.device }} is failing"
description: >-
SMART reports this drive as failing. Check the Drive Health dashboard for
which attribute tripped, and plan a replacement.
# Reallocated, pending or uncorrectable sectors on SATA; a non-empty grown defect
# list on SAS. Not failing yet, so this is a long-fuse warning rather than a page.
- alert: DriveCaution
expr: smartctl:health_code == 1
for: 6h
labels:
severity: warning
annotations:
summary: "{{ $labels.host }}: drive {{ $labels.device }} has degraded sectors"
description: >-
Watch the trend rather than the absolute number -- a stable count is
tolerable, growth is not.
- alert: SelfTestFailed
expr: smart_selftest_last_passed == 0
for: 1h
labels:
severity: critical
annotations:
summary: "{{ $labels.host }}: {{ $labels.device }} failed its {{ $labels.test_type }} self-test"
# Sustained operation above ~55C measurably shortens drive life.
- alert: DriveOverheating
expr: smartctl_device_temperature{temperature_type="current"} > 55
for: 30m
labels:
severity: warning
annotations:
summary: "{{ $labels.host }}: {{ $labels.device }} is at {{ $value }}C"
- name: pools
rules:
- alert: PoolNotOnline
expr: zfs_pool_health_code != 0
for: 5m
labels:
severity: critical
annotations:
summary: "{{ $labels.host }}: pool {{ $labels.pool }} is not ONLINE"
# Errors ZFS could not repair from redundancy: data has been lost and the affected
# files need restoring from backup.
- alert: PoolDataErrors
expr: zfs_pool_data_errors > 0
for: 5m
labels:
severity: critical
annotations:
summary: "{{ $labels.host }}: pool {{ $labels.pool }} has unrepairable data errors"
# ZFS write performance degrades sharply past ~80%, and a pool near 100% becomes
# effectively read-only. 85% leaves room to react.
- alert: PoolNearlyFull
expr: zfs_pool_capacity_ratio > 0.85
for: 1h
labels:
severity: warning
annotations:
summary: "{{ $labels.host }}: pool {{ $labels.pool }} is {{ $value | humanizePercentage }} full"
- alert: ScrubOverdue
expr: time() - zfs_pool_last_scrub_end_timestamp_seconds > 45 * 86400
for: 1h
labels:
severity: warning
annotations:
summary: "{{ $labels.host }}: pool {{ $labels.pool }} has not been scrubbed in 45 days"
# Two intervals late means the auto-snapshot timer has missed at least one run.
- alert: SnapshotsOverdue
expr: >-
(time() - zfs_snapshot_newest_timestamp_seconds)
> on(label) group_left() (2 * zfs_snapshot_interval_seconds)
for: 1h
labels:
severity: warning
annotations:
summary: "{{ $labels.host }}: {{ $labels.label }} snapshots of {{ $labels.dataset }} are overdue"
- name: collection
rules:
# Covers both exporters. Without this, every other alert here goes quiet when a
# machine disappears -- absence of data is not absence of a problem.
- alert: ExporterDown
expr: up == 0
for: 10m
labels:
severity: critical
annotations:
summary: "{{ $labels.host }}: the {{ $labels.job }} exporter is not responding"
# The ZFS collector runs every minute and the SMART one every 15, so a file older
# than 30 minutes means its timer is failing and the dashboards are showing stale
# numbers without saying so.
- alert: CollectorStale
expr: time() - node_textfile_mtime_seconds > 1800
for: 15m
labels:
severity: warning
annotations:
summary: "{{ $labels.host }}: {{ $labels.file }} has not been refreshed"
- alert: TextfileParseError
expr: node_textfile_scrape_error > 0
for: 15m
labels:
severity: warning
annotations:
summary: "{{ $labels.host }}: a collector wrote a malformed .prom file"
|
