diff options
Diffstat (limited to 'rules')
| -rw-r--r-- | rules/alerts.yml | 140 |
1 files changed, 140 insertions, 0 deletions
diff --git a/rules/alerts.yml b/rules/alerts.yml new file mode 100644 index 0000000..cd3b1a7 --- /dev/null +++ b/rules/alerts.yml @@ -0,0 +1,140 @@ +# Alerting rules for the collectors' metrics. +# +# Loaded when services.telemetry.server.alerts.enable is on. Most of the conditions are +# one-liners because rules/smartctl.yml has already reduced drive health to a single +# smartctl:health_code per drive -- see that file for how the verdict is derived. +# +# Every alert carries the `host` label, so notifications say which machine. That label +# comes from the relabelling in modules/server.nix, not from the exporters. +# +# These are deliberately conservative: long `for` durations, and thresholds chosen so a +# healthy fleet is silent. If you want different numbers, set +# services.telemetry.server.alerts.defaultRules = false and point +# services.telemetry.server.prometheus.ruleFiles at your own copy. + +groups: + - name: drives + rules: + # smartctl:health_code is 2 when the drive's own SMART self-assessment failed, an + # attribute is at or below its vendor threshold, or the drive reported errors it + # could not correct. Any of those means replace it. + - alert: DriveFailing + expr: smartctl:health_code == 2 + for: 15m + labels: + severity: critical + annotations: + summary: "{{ $labels.host }}: drive {{ $labels.device }} is failing" + description: >- + SMART reports this drive as failing. Check the Drive Health dashboard for + which attribute tripped, and plan a replacement. + + # Reallocated, pending or uncorrectable sectors on SATA; a non-empty grown defect + # list on SAS. Not failing yet, so this is a long-fuse warning rather than a page. + - alert: DriveCaution + expr: smartctl:health_code == 1 + for: 6h + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: drive {{ $labels.device }} has degraded sectors" + description: >- + Watch the trend rather than the absolute number -- a stable count is + tolerable, growth is not. + + - alert: SelfTestFailed + expr: smart_selftest_last_passed == 0 + for: 1h + labels: + severity: critical + annotations: + summary: "{{ $labels.host }}: {{ $labels.device }} failed its {{ $labels.test_type }} self-test" + + # Sustained operation above ~55C measurably shortens drive life. + - alert: DriveOverheating + expr: smartctl_device_temperature{temperature_type="current"} > 55 + for: 30m + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: {{ $labels.device }} is at {{ $value }}C" + + - name: pools + rules: + - alert: PoolNotOnline + expr: zfs_pool_health_code != 0 + for: 5m + labels: + severity: critical + annotations: + summary: "{{ $labels.host }}: pool {{ $labels.pool }} is not ONLINE" + + # Errors ZFS could not repair from redundancy: data has been lost and the affected + # files need restoring from backup. + - alert: PoolDataErrors + expr: zfs_pool_data_errors > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "{{ $labels.host }}: pool {{ $labels.pool }} has unrepairable data errors" + + # ZFS write performance degrades sharply past ~80%, and a pool near 100% becomes + # effectively read-only. 85% leaves room to react. + - alert: PoolNearlyFull + expr: zfs_pool_capacity_ratio > 0.85 + for: 1h + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: pool {{ $labels.pool }} is {{ $value | humanizePercentage }} full" + + - alert: ScrubOverdue + expr: time() - zfs_pool_last_scrub_end_timestamp_seconds > 45 * 86400 + for: 1h + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: pool {{ $labels.pool }} has not been scrubbed in 45 days" + + # Two intervals late means the auto-snapshot timer has missed at least one run. + - alert: SnapshotsOverdue + expr: >- + (time() - zfs_snapshot_newest_timestamp_seconds) + > on(label) group_left() (2 * zfs_snapshot_interval_seconds) + for: 1h + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: {{ $labels.label }} snapshots of {{ $labels.dataset }} are overdue" + + - name: collection + rules: + # Covers both exporters. Without this, every other alert here goes quiet when a + # machine disappears -- absence of data is not absence of a problem. + - alert: ExporterDown + expr: up == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "{{ $labels.host }}: the {{ $labels.job }} exporter is not responding" + + # The ZFS collector runs every minute and the SMART one every 15, so a file older + # than 30 minutes means its timer is failing and the dashboards are showing stale + # numbers without saying so. + - alert: CollectorStale + expr: time() - node_textfile_mtime_seconds > 1800 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: {{ $labels.file }} has not been refreshed" + + - alert: TextfileParseError + expr: node_textfile_scrape_error > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: a collector wrote a malformed .prom file" |