diff options
| -rw-r--r-- | README.md | 90 | ||||
| -rw-r--r-- | modules/common.nix | 5 | ||||
| -rw-r--r-- | modules/server.nix | 146 | ||||
| -rw-r--r-- | rules/alerts.yml | 140 |
4 files changed, 380 insertions, 1 deletions
@@ -202,6 +202,96 @@ Jobs added through `extraScrapeConfigs` are passed to Prometheus untouched — t --- +## Email alerts + +`alerts.enable` brings up Alertmanager, points Prometheus at it, and loads a set of +alerting rules: + +```nix +services.telemetry.server = { + enable = true; + collectorHosts = [ "poweredge" "optiplex" ]; + + alerts = { + enable = true; + to = [ "you@example.com" ]; + from = "alerts@example.com"; + smarthost = "smtp.fastmail.com:587"; + username = "alerts@example.com"; + passwordFile = config.sops.secrets.smtp-password.path; + }; +}; +``` + +`passwordFile` is handed to Alertmanager as `smtp_auth_password_file` and read at +runtime, so the password never enters the Nix store. Drop `username` and `passwordFile` +for an unauthenticated local relay. + +### What it alerts on + +Shipped in `rules/alerts.yml`, loaded unless you set `alerts.defaultRules = false`: + +| | condition | severity | +|---|---|---| +| `DriveFailing` | SMART self-assessment failed, an attribute is below its threshold, or uncorrectable errors | critical | +| `DriveCaution` | reallocated/pending sectors, or a non-empty SAS grown defect list | warning | +| `SelfTestFailed` | last short or extended self-test did not pass | critical | +| `DriveOverheating` | above 55 °C for 30 minutes | warning | +| `PoolNotOnline` | pool is DEGRADED, FAULTED, … | critical | +| `PoolDataErrors` | errors ZFS could not repair from redundancy | critical | +| `PoolNearlyFull` | over 85% | warning | +| `ScrubOverdue` | no completed scrub in 45 days | warning | +| `SnapshotsOverdue` | auto-snapshot timer has missed two intervals | warning | +| `ExporterDown` | `up == 0` for 10 minutes | critical | +| `CollectorStale` | a `.prom` file has not been refreshed in 30 minutes | warning | +| `TextfileParseError` | node\_exporter could not parse a collector's output | warning | + +Most are one-liners because `rules/smartctl.yml` has already reduced drive health to a +single `smartctl:health_code` per drive. Every alert carries the `host` label, so the +mail tells you which machine. + +`ExporterDown` matters more than it looks: without it, every other alert here goes quiet +when a machine disappears — absence of data is not absence of a problem. + +### Changing the rules + +Thresholds are not options. Copy the file, edit it, and swap it in: + +```nix +services.telemetry.server = { + alerts.defaultRules = false; + prometheus.ruleFiles = [ + "${inputs.telemetry}/rules/smartctl.yml" # keep: the health verdict lives here + ./my-alerts.yml + ]; +}; +``` + +`promtool` runs over whatever you supply at build time, so a bad expression fails +`nixos-rebuild` rather than at 3am. + +For routing — inhibit rules, a second receiver, severity-based routes — use +`alerts.settings`, which is merged over the generated Alertmanager config: + +```nix +alerts.settings.route.routes = [{ + matchers = [ "severity=\"warning\"" ]; + receiver = "email"; + repeat_interval = "7d"; # nag less about warnings +}]; +``` + +### Notes + +- Alertmanager's port is opened by `openFirewall` along with Prometheus'. Its UI is where + you silence alerts — and anyone who can reach it can silence anything, same trust model + as the Prometheus port. +- Alerts are grouped by `alertname` and `host`, so nine failing drives on one machine + arrive as one mail. `alerts.repeatInterval` (default 24h) decides how often a + still-firing alert nags you. +- Alertmanager runs `envsubst` over its config at start, so a literal `${...}` in any + string you add via `alerts.settings` will be substituted. + ## Layout ``` diff --git a/modules/common.nix b/modules/common.nix index 480aa62..fd343de 100644 --- a/modules/common.nix +++ b/modules/common.nix @@ -38,6 +38,11 @@ in Port Prometheus listens on. ''; + alertmanager = portOption 9093 '' + Port Alertmanager listens on. Only Prometheus talks to it, but the web UI is + where you silence alerts. + ''; + grafana = portOption 3000 '' Port Grafana listens on. If the reverse proxy is enabled this stays internal and the proxy is what you connect to. diff --git a/modules/server.nix b/modules/server.nix index d9449a8..c6094ef 100644 --- a/modules/server.nix +++ b/modules/server.nix @@ -250,6 +250,90 @@ in }; }; + alerts = { + enable = mkEnableOption '' + Alertmanager, wired to Prometheus and delivering by email. + + The alerting rules themselves are plain PromQL in rules/alerts.yml, checked by + promtool at build time like everything else here -- so a typo in an expression + fails `nixos-rebuild`, not at 3am + ''; + + defaultRules = mkOption { + type = types.bool; + default = true; + description = '' + Load the alerting rules shipped with this module: drive failing or degraded, + failed self-test, drive overheating, pool not ONLINE, unrepairable data + errors, pool over 85% full, scrub or snapshots overdue, exporter down, and a + collector that has stopped refreshing its file. + + Turn this off and point + {option}`services.telemetry.server.prometheus.ruleFiles` at your own copy to + change the thresholds. + ''; + }; + + to = mkOption { + type = types.listOf types.str; + example = [ "you@example.com" ]; + description = "Where notifications are sent."; + }; + + from = mkOption { + type = types.str; + example = "alerts@example.com"; + description = "Envelope sender. Many providers require this to be the account you authenticate as."; + }; + + smarthost = mkOption { + type = types.str; + example = "smtp.fastmail.com:587"; + description = "SMTP server and port to relay through."; + }; + + username = mkOption { + type = types.nullOr types.str; + default = null; + description = "SMTP username. Null sends unauthenticated, which only works for a local relay."; + }; + + passwordFile = mkOption { + type = types.nullOr types.path; + default = null; + example = lib.literalExpression "config.sops.secrets.smtp-password.path"; + description = '' + File containing the SMTP password. Passed to Alertmanager as + `smtp_auth_password_file` and read at runtime, so it never enters the Nix + store. Must be readable by the alertmanager user. + ''; + }; + + requireTLS = mkOption { + type = types.bool; + default = true; + description = "Require STARTTLS. Turn off only for a local relay on loopback."; + }; + + repeatInterval = mkOption { + type = types.str; + default = "24h"; + description = '' + How long before an alert that is still firing is re-sent. A drive does not + stop failing, so this is the knob that decides how often it nags you. + ''; + }; + + settings = mkOption { + type = types.attrs; + default = { }; + description = '' + Merged over the generated Alertmanager configuration, last wins. The escape + hatch for inhibit rules, extra receivers, or routing by severity. + ''; + }; + }; + proxy = { enable = mkOption { type = types.bool; @@ -287,6 +371,14 @@ in ''; } { + assertion = !cfg.alerts.enable || cfg.prometheus.enable; + message = '' + services.telemetry.server.alerts.enable requires prometheus.enable -- the + alerting rules are evaluated by Prometheus, which then pushes to + Alertmanager. + ''; + } + { assertion = !cfg.grafana.enable || secretKeyPath != null; message = '' Grafana needs a secret key. Either leave @@ -348,7 +440,8 @@ in scrape_timeout = "10s"; }; - inherit (cfg.prometheus) ruleFiles; + ruleFiles = cfg.prometheus.ruleFiles + ++ lib.optional (cfg.alerts.enable && cfg.alerts.defaultRules) ../rules/alerts.yml; scrapeConfigs = optionals (cfg.prometheus.nodeTargets != [ ]) [{ @@ -364,6 +457,54 @@ in static_configs = [{ targets = cfg.prometheus.smartctlTargets; }]; }] ++ cfg.prometheus.extraScrapeConfigs; + + # Alertmanager lives inside this block rather than in a `services.prometheus.*` + # definition of its own: Nix will not let one attrset both define + # `services.prometheus` wholesale and add a path underneath it. + alertmanagers = mkIf cfg.alerts.enable [{ + static_configs = [{ targets = [ "127.0.0.1:${toString ports.alertmanager}" ]; }]; + }]; + + alertmanager = mkIf cfg.alerts.enable { + enable = true; + port = ports.alertmanager; + listenAddress = cfg.listenAddress; + configuration = lib.recursiveUpdate + { + global = { + smtp_smarthost = cfg.alerts.smarthost; + smtp_from = cfg.alerts.from; + smtp_require_tls = cfg.alerts.requireTLS; + } + // lib.optionalAttrs (cfg.alerts.username != null) { + smtp_auth_username = cfg.alerts.username; + } + // lib.optionalAttrs (cfg.alerts.passwordFile != null) { + smtp_auth_password_file = toString cfg.alerts.passwordFile; + }; + + route = { + receiver = "email"; + # By alertname and host, so nine failing drives on one machine arrive as one + # mail rather than nine. + group_by = [ "alertname" "host" ]; + group_wait = "1m"; + group_interval = "10m"; + repeat_interval = cfg.alerts.repeatInterval; + }; + + receivers = [{ + name = "email"; + email_configs = map + (addr: { + to = addr; + send_resolved = true; + }) + cfg.alerts.to; + }]; + } + cfg.alerts.settings; + }; }; # -------------------------------------------------------------------------------- @@ -481,6 +622,9 @@ in networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall ( optionals cfg.loki.enable [ ports.loki ] ++ optionals cfg.prometheus.enable [ ports.prometheus ] + # The Alertmanager UI is how you silence an alert. Note anyone who can reach it + # can silence anything -- same trust model as the Prometheus port above. + ++ optionals cfg.alerts.enable [ ports.alertmanager ] ++ (if cfg.proxy.enable then [ 80 443 ] else optionals cfg.grafana.enable [ ports.grafana ]) ); }; diff --git a/rules/alerts.yml b/rules/alerts.yml new file mode 100644 index 0000000..cd3b1a7 --- /dev/null +++ b/rules/alerts.yml @@ -0,0 +1,140 @@ +# Alerting rules for the collectors' metrics. +# +# Loaded when services.telemetry.server.alerts.enable is on. Most of the conditions are +# one-liners because rules/smartctl.yml has already reduced drive health to a single +# smartctl:health_code per drive -- see that file for how the verdict is derived. +# +# Every alert carries the `host` label, so notifications say which machine. That label +# comes from the relabelling in modules/server.nix, not from the exporters. +# +# These are deliberately conservative: long `for` durations, and thresholds chosen so a +# healthy fleet is silent. If you want different numbers, set +# services.telemetry.server.alerts.defaultRules = false and point +# services.telemetry.server.prometheus.ruleFiles at your own copy. + +groups: + - name: drives + rules: + # smartctl:health_code is 2 when the drive's own SMART self-assessment failed, an + # attribute is at or below its vendor threshold, or the drive reported errors it + # could not correct. Any of those means replace it. + - alert: DriveFailing + expr: smartctl:health_code == 2 + for: 15m + labels: + severity: critical + annotations: + summary: "{{ $labels.host }}: drive {{ $labels.device }} is failing" + description: >- + SMART reports this drive as failing. Check the Drive Health dashboard for + which attribute tripped, and plan a replacement. + + # Reallocated, pending or uncorrectable sectors on SATA; a non-empty grown defect + # list on SAS. Not failing yet, so this is a long-fuse warning rather than a page. + - alert: DriveCaution + expr: smartctl:health_code == 1 + for: 6h + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: drive {{ $labels.device }} has degraded sectors" + description: >- + Watch the trend rather than the absolute number -- a stable count is + tolerable, growth is not. + + - alert: SelfTestFailed + expr: smart_selftest_last_passed == 0 + for: 1h + labels: + severity: critical + annotations: + summary: "{{ $labels.host }}: {{ $labels.device }} failed its {{ $labels.test_type }} self-test" + + # Sustained operation above ~55C measurably shortens drive life. + - alert: DriveOverheating + expr: smartctl_device_temperature{temperature_type="current"} > 55 + for: 30m + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: {{ $labels.device }} is at {{ $value }}C" + + - name: pools + rules: + - alert: PoolNotOnline + expr: zfs_pool_health_code != 0 + for: 5m + labels: + severity: critical + annotations: + summary: "{{ $labels.host }}: pool {{ $labels.pool }} is not ONLINE" + + # Errors ZFS could not repair from redundancy: data has been lost and the affected + # files need restoring from backup. + - alert: PoolDataErrors + expr: zfs_pool_data_errors > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "{{ $labels.host }}: pool {{ $labels.pool }} has unrepairable data errors" + + # ZFS write performance degrades sharply past ~80%, and a pool near 100% becomes + # effectively read-only. 85% leaves room to react. + - alert: PoolNearlyFull + expr: zfs_pool_capacity_ratio > 0.85 + for: 1h + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: pool {{ $labels.pool }} is {{ $value | humanizePercentage }} full" + + - alert: ScrubOverdue + expr: time() - zfs_pool_last_scrub_end_timestamp_seconds > 45 * 86400 + for: 1h + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: pool {{ $labels.pool }} has not been scrubbed in 45 days" + + # Two intervals late means the auto-snapshot timer has missed at least one run. + - alert: SnapshotsOverdue + expr: >- + (time() - zfs_snapshot_newest_timestamp_seconds) + > on(label) group_left() (2 * zfs_snapshot_interval_seconds) + for: 1h + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: {{ $labels.label }} snapshots of {{ $labels.dataset }} are overdue" + + - name: collection + rules: + # Covers both exporters. Without this, every other alert here goes quiet when a + # machine disappears -- absence of data is not absence of a problem. + - alert: ExporterDown + expr: up == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "{{ $labels.host }}: the {{ $labels.job }} exporter is not responding" + + # The ZFS collector runs every minute and the SMART one every 15, so a file older + # than 30 minutes means its timer is failing and the dashboards are showing stale + # numbers without saying so. + - alert: CollectorStale + expr: time() - node_textfile_mtime_seconds > 1800 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: {{ $labels.file }} has not been refreshed" + + - alert: TextfileParseError + expr: node_textfile_scrape_error > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $labels.host }}: a collector wrote a malformed .prom file" |