# Alerting rules for the collectors' metrics.
#
# Loaded when services.telemetry.server.alerts.enable is on. Most of the conditions are
# one-liners because rules/smartctl.yml has already reduced drive health to a single
# smartctl:health_code per drive -- see that file for how the verdict is derived.
#
# Every alert carries the `host` label, so notifications say which machine. That label
# comes from the relabelling in modules/server.nix, not from the exporters.
#
# These are deliberately conservative: long `for` durations, and thresholds chosen so a
# healthy fleet is silent. If you want different numbers, set
# services.telemetry.server.alerts.defaultRules = false and point
# services.telemetry.server.prometheus.ruleFiles at your own copy.

groups:
  - name: drives
    rules:
      # smartctl:health_code is 2 when the drive's own SMART self-assessment failed, an
      # attribute is at or below its vendor threshold, or the drive reported errors it
      # could not correct. Any of those means replace it.
      - alert: DriveFailing
        expr: smartctl:health_code == 2
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.host }}: drive {{ $labels.device }} is failing"
          description: >-
            SMART reports this drive as failing. Check the Drive Health dashboard for
            which attribute tripped, and plan a replacement.

      # Reallocated, pending or uncorrectable sectors on SATA; a non-empty grown defect
      # list on SAS. Not failing yet, so this is a long-fuse warning rather than a page.
      - alert: DriveCaution
        expr: smartctl:health_code == 1
        for: 6h
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.host }}: drive {{ $labels.device }} has degraded sectors"
          description: >-
            Watch the trend rather than the absolute number -- a stable count is
            tolerable, growth is not.

      - alert: SelfTestFailed
        expr: smart_selftest_last_passed == 0
        for: 1h
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.host }}: {{ $labels.device }} failed its {{ $labels.test_type }} self-test"

      # Sustained operation above ~55C measurably shortens drive life.
      - alert: DriveOverheating
        expr: smartctl_device_temperature{temperature_type="current"} > 55
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.host }}: {{ $labels.device }} is at {{ $value }}C"

  - name: pools
    rules:
      - alert: PoolNotOnline
        expr: zfs_pool_health_code != 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.host }}: pool {{ $labels.pool }} is not ONLINE"

      # Errors ZFS could not repair from redundancy: data has been lost and the affected
      # files need restoring from backup.
      - alert: PoolDataErrors
        expr: zfs_pool_data_errors > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.host }}: pool {{ $labels.pool }} has unrepairable data errors"

      # ZFS write performance degrades sharply past ~80%, and a pool near 100% becomes
      # effectively read-only. 85% leaves room to react.
      - alert: PoolNearlyFull
        expr: zfs_pool_capacity_ratio > 0.85
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.host }}: pool {{ $labels.pool }} is {{ $value | humanizePercentage }} full"

      - alert: ScrubOverdue
        expr: time() - zfs_pool_last_scrub_end_timestamp_seconds > 45 * 86400
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.host }}: pool {{ $labels.pool }} has not been scrubbed in 45 days"

      # Two intervals late means the auto-snapshot timer has missed at least one run.
      - alert: SnapshotsOverdue
        expr: >-
          (time() - zfs_snapshot_newest_timestamp_seconds)
          > on(label) group_left() (2 * zfs_snapshot_interval_seconds)
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.host }}: {{ $labels.label }} snapshots of {{ $labels.dataset }} are overdue"

  - name: collection
    rules:
      # Covers both exporters. Without this, every other alert here goes quiet when a
      # machine disappears -- absence of data is not absence of a problem.
      - alert: ExporterDown
        expr: up == 0
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.host }}: the {{ $labels.job }} exporter is not responding"

      # The ZFS collector runs every minute and the SMART one every 15, so a file older
      # than 30 minutes means its timer is failing and the dashboards are showing stale
      # numbers without saying so.
      - alert: CollectorStale
        expr: time() - node_textfile_mtime_seconds > 1800
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.host }}: {{ $labels.file }} has not been refreshed"

      - alert: TextfileParseError
        expr: node_textfile_scrape_error > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.host }}: a collector wrote a malformed .prom file"