path: root/modules/collectors.nix

# Host-side telemetry sources.
#
# Everything here has to run on the host, not in a container: it reads host hardware and
# kernel state -- /proc, /sys, SMART over raw block devices, ZFS ioctls -- that a
# container has no view of. The half that can be containerised is modules/server.nix.
{ config, lib, pkgs, ... }:

let
  inherit (lib) mkIf mkOption mkEnableOption mkForce types optionals concatMap;

  cfg = config.services.telemetry.collectors;
  ports = config.services.telemetry.ports;

  collectorPkgs = pkgs.callPackages ./packages.nix {
    # Match the userland to the running kernel module.
    zfs = config.boot.zfs.package;
  };

  # services.zfs.autoSnapshot's per-label keep counts, handed to the collector so it can
  # report how far each label is from its target rather than just how many snapshots
  # exist. Silently empty when auto-snapshotting is off, which the collector handles.
  snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ];

  retentionArgs = optionals config.services.zfs.autoSnapshot.enable (
    concatMap
      (label: [
        "--retention"
        "${label}=${toString config.services.zfs.autoSnapshot.${label}}"
      ])
      snapshotLabels
  );

  # Hardening common to both collectors. Each writes exactly one file into the textfile
  # directory and talks to nothing over the network.
  hardening = {
    LockPersonality = true;
    MemoryDenyWriteExecute = true;
    NoNewPrivileges = true;
    PrivateNetwork = true;
    PrivateTmp = true;
    ProtectControlGroups = true;
    ProtectHome = true;
    ProtectHostname = true;
    ProtectKernelLogs = true;
    ProtectKernelModules = true;
    ProtectSystem = "strict";
    RestrictAddressFamilies = [ "AF_UNIX" ];
    RestrictNamespaces = true;
    RestrictRealtime = true;
    RestrictSUIDSGID = true;
    SystemCallArchitectures = "native";
  };

  mkCollectorService = { description, execStart, extraServiceConfig ? { } }: {
    inherit description;
    serviceConfig = hardening // {
      Type = "oneshot";
      ExecStart = execStart;
      User = "root";
      StateDirectory = "prometheus-node-exporter-textfile";
      StateDirectoryMode = "0755";
      # node_exporter runs as a different user and has to be able to read what we write.
      UMask = "0022";
      TimeoutStartSec = "180s";
    } // extraServiceConfig;
  };

  mkCollectorTimer = { description, onBootSec, onUnitActiveSec, accuracySec, unit }: {
    inherit description;
    wantedBy = [ "timers.target" ];
    timerConfig = {
      OnBootSec = onBootSec;
      OnUnitActiveSec = onUnitActiveSec;
      AccuracySec = accuracySec;
      Unit = unit;
    };
  };
in
{
  imports = [ ./common.nix ];

  options.services.telemetry.collectors = {
    enable = mkEnableOption "host-side telemetry collectors (node exporter, SMART, ZFS)";

    listenAddress = mkOption {
      type = types.str;
      default = "0.0.0.0";
      description = ''
        Address the exporters bind to. The default is deliberately permissive because
        the whole point of splitting the halves is that the server scrapes these from
        somewhere else -- a container on a bridge, or another machine. Narrow it if the
        server runs on this host.
      '';
    };

    openFirewall = mkOption {
      type = types.bool;
      default = true;
      description = ''
        Open the node and smartctl exporter ports. Needed whenever the server is not on
        this host's loopback, which includes a container with `privateNetwork`.
      '';
    };

    textfileDir = mkOption {
      type = types.path;
      default = "/var/lib/prometheus-node-exporter-textfile";
      description = ''
        Directory the ZFS and SMART collectors write `.prom` files into, and that
        node_exporter's textfile collector reads. Changing this also changes the
        StateDirectory of both collector units.
      '';
    };

    node = {
      enable = mkOption {
        type = types.bool;
        default = true;
        description = "Run the Prometheus node exporter.";
      };

      extraCollectors = mkOption {
        type = types.listOf types.str;
        default = [ ];
        example = [ "interrupts" ];
        description = ''
          Additional node_exporter collectors, on top of the systemd, processes and
          textfile collectors this module enables.

          Note this *adds* to node_exporter's defaults -- cpu, diskstats, filesystem,
          meminfo, zfs, hwmon and pressure are already on, so listing those does
          nothing. `interrupts` is the notable opt-in: it fills the Node Exporter Full
          "IRQ Detail" panel but emits one series per (cpu, irq) pair, which is
          thousands of series on a many-core box.
        '';
      };
    };

    smart = {
      enable = mkOption {
        type = types.bool;
        default = true;
        description = ''
          Run the smartctl exporter and the self-test history collector.
        '';
      };

      devices = mkOption {
        type = types.listOf types.str;
        default = [ ];
        example = [ "/dev/sda" "/dev/nvme0" "/dev/bus/0;megaraid,1" ];
        description = ''
          Devices to poll, shared by the smartctl exporter and the self-test collector.
          Empty means auto-discovery via `smartctl --scan`, which is correct for
          direct-attached AHCI/HBA disks. Behind a RAID controller `--scan` returns
          nothing useful and you have to list the drives explicitly, giving each one its
          `-d` type after a semicolon: `"/dev/bus/0;megaraid,1"`.

          Note that setting this also turns off the exporter's periodic rescan, so
          hot-swapped drives will not be picked up until it restarts.
        '';
      };

      maxInterval = mkOption {
        type = types.str;
        default = "60s";
        description = ''
          Floor on how often the exporter actually shells out to smartctl, independent
          of how often Prometheus scrapes it. SMART data moves slowly and polling can
          spin up idle disks.
        '';
      };

      selfTestInterval = mkOption {
        type = types.str;
        default = "15min";
        description = ''
          How often to re-read the self-test logs. The data only changes when a test
          finishes, so this can be far slower than the exporter's own polling.
        '';
      };
    };

    zfs = {
      enable = mkOption {
        type = types.bool;
        default = config.boot.zfs.enabled;
        defaultText = lib.literalExpression "config.boot.zfs.enabled";
        description = ''
          Collect pool capacity and health, vdev errors, scrub history, dataset
          properties and snapshot retention from the zpool/zfs CLIs.
        '';
      };

      interval = mkOption {
        type = types.str;
        default = "1min";
        description = ''
          How often to re-read pool and dataset state. Listing every snapshot on a busy
          pool is the slow part; if this starts overlapping its own runs, raise it.
        '';
      };
    };

    logs = {
      enable = mkOption {
        type = types.bool;
        default = true;
        description = "Ship the systemd journal to Loki with Grafana Alloy.";
      };

      lokiUrl = mkOption {
        type = types.str;
        default = "http://localhost:${toString ports.loki}";
        defaultText = lib.literalExpression ''"http://localhost:''${toString config.services.telemetry.ports.loki}"'';
        example = "http://metrics.lan:3100";
        description = ''
          Base URL of the Loki instance to push to. Point this at wherever
          modules/server.nix runs; the `/loki/api/v1/push` path is appended.
        '';
      };

      maxAge = mkOption {
        type = types.str;
        default = "12h";
        description = ''
          How far back into the journal to read on startup. Anything older than this is
          skipped rather than replayed into Loki.
        '';
      };
    };
  };

  config = mkIf cfg.enable {
    # ------------------------------------------------------------------------------
    # node exporter
    # ------------------------------------------------------------------------------
    services.prometheus.exporters.node = mkIf cfg.node.enable {
      enable = true;
      port = ports.node;
      inherit (cfg) listenAddress;

      enabledCollectors = [
        "systemd" # node_systemd_* — the "Systemd" row, and the zfs-* timer panels
        "processes" # node_processes_* — "Processes Detailed States", "PIDs", "Threads"
      ] ++ cfg.node.extraCollectors;

      extraFlags = [
        "--collector.textfile.directory=${cfg.textfileDir}"
        # node_systemd_unit_start_time_seconds, so the dashboards can show when the
        # zfs-snapshot-* and zfs-scrub units last ran rather than only whether they
        # failed.
        "--collector.systemd.enable-start-time-metrics"
      ];
    };

    # Upstream hardens the unit with ProtectHome=true, which makes /home look empty and
    # blanks out the filesystem panels for any dataset mounted under it. read-only still
    # allows statfs while keeping the sandbox.
    systemd.services.prometheus-node-exporter = mkIf cfg.node.enable {
      serviceConfig.ProtectHome = mkForce "read-only";
    };

    # ------------------------------------------------------------------------------
    # SMART
    # ------------------------------------------------------------------------------
    services.prometheus.exporters.smartctl = mkIf cfg.smart.enable {
      enable = true;
      port = ports.smartctl;
      inherit (cfg) listenAddress;
      inherit (cfg.smart) devices maxInterval;
    };

    systemd.services.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorService {
      description = "Collect SMART self-test history and drive identity";
      # The same device list the exporter gets. Without this the collector would fall
      # back to its own `smartctl --scan`, which is exactly what cannot see the drives
      # in the one configuration the option exists for -- so every row title would read
      # "identity pending" and self-test history would be empty.
      execStart = lib.escapeShellArgs ([
        (lib.getExe collectorPkgs.smart-selftest)
        "--output"
        "${cfg.textfileDir}/smart-selftest.prom"
      ] ++ concatMap (device: [ "--device" device ]) cfg.smart.devices);
      # Same shape as the smartctl exporter's own hardening: raw device access to talk
      # to the disks, and nothing else.
      extraServiceConfig = {
        AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
        CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
        DevicePolicy = "closed";
        DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ];
      };
    });

    systemd.timers.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorTimer {
      description = "Collect SMART self-test history periodically";
      onBootSec = "3min";
      onUnitActiveSec = cfg.smart.selfTestInterval;
      accuracySec = "1min";
      unit = "smart-selftest-collector.service";
    });

    # ------------------------------------------------------------------------------
    # ZFS
    # ------------------------------------------------------------------------------
    systemd.services.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorService {
      description = "Collect ZFS pool, dataset and snapshot metrics";
      execStart = lib.escapeShellArgs ([
        (lib.getExe collectorPkgs.zfs-metrics)
        "--output"
        "${cfg.textfileDir}/zfs.prom"
      ] ++ retentionArgs);
      # CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated on it;
      # everything else is dropped.
      extraServiceConfig = {
        CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ];
        SystemCallFilter = [ "@system-service" ];
      };
    });

    systemd.timers.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorTimer {
      description = "Collect ZFS metrics periodically";
      onBootSec = "1min";
      onUnitActiveSec = cfg.zfs.interval;
      accuracySec = "5s";
      unit = "zfs-textfile-collector.service";
    });

    # An earlier version of the ZFS collector wrote zpool.prom. Nothing removes a stale
    # textfile once its writer is gone, and node_exporter will happily keep serving
    # those frozen metrics forever, so delete it on activation.
    systemd.tmpfiles.rules = mkIf cfg.zfs.enable [
      "r ${cfg.textfileDir}/zpool.prom"
    ];

    # ------------------------------------------------------------------------------
    # Journal shipping
    # ------------------------------------------------------------------------------
    services.alloy.enable = mkIf cfg.logs.enable true;

    environment.etc."alloy/config.alloy" = mkIf cfg.logs.enable {
      text = ''
        loki.source.journal "journal" {
          forward_to    = [loki.write.default.receiver]
          relabel_rules = loki.relabel.journal.rules
          max_age       = "${cfg.logs.maxAge}"
        }

        loki.relabel "journal" {
          forward_to = []
          rule {
            source_labels = ["__journal__systemd_unit"]
            target_label  = "unit"
          }
          rule {
            source_labels = ["__journal__hostname"]
            target_label  = "host"
          }
          rule {
            source_labels = ["__journal_priority_keyword"]
            target_label  = "level"
          }
        }

        loki.write "default" {
          endpoint {
            url = "${cfg.logs.lokiUrl}/loki/api/v1/push"
          }
        }
      '';
    };

    # ------------------------------------------------------------------------------
    # Firewall
    # ------------------------------------------------------------------------------
    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall (
      optionals cfg.node.enable [ ports.node ]
      ++ optionals cfg.smart.enable [ ports.smartctl ]
    );
  };
}