split into a flake with separable collectors/server modules

Replaces the single lgtm.nix with a flake exposing two NixOS modules under
services.telemetry, so the web services can move into a container declared in
the consuming config rather than one hardcoded here.

  modules/collectors.nix  host-side: node + smartctl exporters, the ZFS and
                          SMART textfile collector timers, Alloy journal
                          shipping. Must run on the host -- reads /proc, /sys,
                          raw block devices and ZFS ioctls.
  modules/server.nix      Prometheus, Loki, Grafana, optional Caddy.
  modules/common.nix      ports, shared by both halves.
  modules/dashboards.nix  jq fixup and grafana.com fetches.
  modules/packages.nix    the two collectors as ordinary packages.

Modules are exported as paths, not functions, so they evaluate with the
consumer's nixpkgs; the flake's own input only affects packages/checks/devShells.

Multi-host support. collectorHosts is a list, both scrape jobs relabel
__address__ into a `host` label, and all 133 queries in the two generated
dashboards are scoped to host=~"$host" behind a single-select Host picker.
`instance` could not serve as that key: one machine is scraped twice on
different ports (textfile metrics on :9100, smartctl_* on :9633) and the
recording rules join across that boundary.

Grafana's secret key becomes a plain secretKeyFile path wired through
LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No
sops-nix input.

nix flake check runs promtool over the rules, diffs the committed dashboard
JSON against what the generators produce, and evaluates both modules.

Also fixes smart-selftest emitting a placeholder identity line when smartctl
cannot open a device. smartctl exits 0 with well-formed but empty JSON in that
case, so "SSD - unknown model - no serial" was reaching a dashboard row title;
it now emits nothing and lets the smartctl:ident fallback apply.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ

1 files changed, 368 insertions, 0 deletions

diff --git a/modules/collectors.nix b/modules/collectors.nix
new file mode 100644
index 0000000..6f328ee
--- /dev/null
+++ b/modules/collectors.nix
@@ -0,0 +1,368 @@
+# Host-side telemetry sources.
+#
+# Everything here has to run on the host, not in a container: it reads host hardware and
+# kernel state -- /proc, /sys, SMART over raw block devices, ZFS ioctls -- that a
+# container has no view of. The half that can be containerised is modules/server.nix.
+{ config, lib, pkgs, ... }:
+
+let
+  inherit (lib) mkIf mkOption mkEnableOption mkForce types optionals concatMap;
+
+  cfg = config.services.telemetry.collectors;
+  ports = config.services.telemetry.ports;
+
+  collectorPkgs = pkgs.callPackages ./packages.nix {
+    # Match the userland to the running kernel module.
+    zfs = config.boot.zfs.package;
+  };
+
+  # services.zfs.autoSnapshot's per-label keep counts, handed to the collector so it can
+  # report how far each label is from its target rather than just how many snapshots
+  # exist. Silently empty when auto-snapshotting is off, which the collector handles.
+  snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ];
+
+  retentionArgs = optionals config.services.zfs.autoSnapshot.enable (
+    concatMap
+      (label: [
+        "--retention"
+        "${label}=${toString config.services.zfs.autoSnapshot.${label}}"
+      ])
+      snapshotLabels
+  );
+
+  # Hardening common to both collectors. Each writes exactly one file into the textfile
+  # directory and talks to nothing over the network.
+  hardening = {
+    LockPersonality = true;
+    MemoryDenyWriteExecute = true;
+    NoNewPrivileges = true;
+    PrivateNetwork = true;
+    PrivateTmp = true;
+    ProtectControlGroups = true;
+    ProtectHome = true;
+    ProtectHostname = true;
+    ProtectKernelLogs = true;
+    ProtectKernelModules = true;
+    ProtectSystem = "strict";
+    RestrictAddressFamilies = [ "AF_UNIX" ];
+    RestrictNamespaces = true;
+    RestrictRealtime = true;
+    RestrictSUIDSGID = true;
+    SystemCallArchitectures = "native";
+  };
+
+  mkCollectorService = { description, execStart, extraServiceConfig ? { } }: {
+    inherit description;
+    serviceConfig = hardening // {
+      Type = "oneshot";
+      ExecStart = execStart;
+      User = "root";
+      StateDirectory = "prometheus-node-exporter-textfile";
+      StateDirectoryMode = "0755";
+      # node_exporter runs as a different user and has to be able to read what we write.
+      UMask = "0022";
+      TimeoutStartSec = "180s";
+    } // extraServiceConfig;
+  };
+
+  mkCollectorTimer = { description, onBootSec, onUnitActiveSec, accuracySec, unit }: {
+    inherit description;
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnBootSec = onBootSec;
+      OnUnitActiveSec = onUnitActiveSec;
+      AccuracySec = accuracySec;
+      Unit = unit;
+    };
+  };
+in
+{
+  imports = [ ./common.nix ];
+
+  options.services.telemetry.collectors = {
+    enable = mkEnableOption "host-side telemetry collectors (node exporter, SMART, ZFS)";
+
+    listenAddress = mkOption {
+      type = types.str;
+      default = "0.0.0.0";
+      description = ''
+        Address the exporters bind to. The default is deliberately permissive because
+        the whole point of splitting the halves is that the server scrapes these from
+        somewhere else -- a container on a bridge, or another machine. Narrow it if the
+        server runs on this host.
+      '';
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Open the node and smartctl exporter ports. Needed whenever the server is not on
+        this host's loopback, which includes a container with `privateNetwork`.
+      '';
+    };
+
+    textfileDir = mkOption {
+      type = types.path;
+      default = "/var/lib/prometheus-node-exporter-textfile";
+      description = ''
+        Directory the ZFS and SMART collectors write `.prom` files into, and that
+        node_exporter's textfile collector reads. Changing this also changes the
+        StateDirectory of both collector units.
+      '';
+    };
+
+    node = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run the Prometheus node exporter.";
+      };
+
+      extraCollectors = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        example = [ "interrupts" ];
+        description = ''
+          Additional node_exporter collectors, on top of the systemd, processes and
+          textfile collectors this module enables.
+
+          Note this *adds* to node_exporter's defaults -- cpu, diskstats, filesystem,
+          meminfo, zfs, hwmon and pressure are already on, so listing those does
+          nothing. `interrupts` is the notable opt-in: it fills the Node Exporter Full
+          "IRQ Detail" panel but emits one series per (cpu, irq) pair, which is
+          thousands of series on a many-core box.
+        '';
+      };
+    };
+
+    smart = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Run the smartctl exporter and the self-test history collector.
+        '';
+      };
+
+      devices = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        example = [ "/dev/sda" "/dev/nvme0" ];
+        description = ''
+          Devices to poll. Empty means auto-discovery via `smartctl --scan`, which is
+          correct for direct-attached AHCI/HBA disks. Behind a RAID controller you
+          generally have to list devices explicitly with their `-d` type.
+        '';
+      };
+
+      maxInterval = mkOption {
+        type = types.str;
+        default = "60s";
+        description = ''
+          Floor on how often the exporter actually shells out to smartctl, independent
+          of how often Prometheus scrapes it. SMART data moves slowly and polling can
+          spin up idle disks.
+        '';
+      };
+
+      selfTestInterval = mkOption {
+        type = types.str;
+        default = "15min";
+        description = ''
+          How often to re-read the self-test logs. The data only changes when a test
+          finishes, so this can be far slower than the exporter's own polling.
+        '';
+      };
+    };
+
+    zfs = {
+      enable = mkOption {
+        type = types.bool;
+        default = config.boot.zfs.enabled;
+        defaultText = lib.literalExpression "config.boot.zfs.enabled";
+        description = ''
+          Collect pool capacity and health, vdev errors, scrub history, dataset
+          properties and snapshot retention from the zpool/zfs CLIs.
+        '';
+      };
+
+      interval = mkOption {
+        type = types.str;
+        default = "1min";
+        description = ''
+          How often to re-read pool and dataset state. Listing every snapshot on a busy
+          pool is the slow part; if this starts overlapping its own runs, raise it.
+        '';
+      };
+    };
+
+    logs = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Ship the systemd journal to Loki with Grafana Alloy.";
+      };
+
+      lokiUrl = mkOption {
+        type = types.str;
+        default = "http://localhost:${toString ports.loki}";
+        defaultText = lib.literalExpression ''"http://localhost:''${toString config.services.telemetry.ports.loki}"'';
+        example = "http://metrics.lan:3100";
+        description = ''
+          Base URL of the Loki instance to push to. Point this at wherever
+          modules/server.nix runs; the `/loki/api/v1/push` path is appended.
+        '';
+      };
+
+      maxAge = mkOption {
+        type = types.str;
+        default = "12h";
+        description = ''
+          How far back into the journal to read on startup. Anything older than this is
+          skipped rather than replayed into Loki.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # ------------------------------------------------------------------------------
+    # node exporter
+    # ------------------------------------------------------------------------------
+    services.prometheus.exporters.node = mkIf cfg.node.enable {
+      enable = true;
+      port = ports.node;
+      inherit (cfg) listenAddress;
+
+      enabledCollectors = [
+        "systemd" # node_systemd_* — the "Systemd" row, and the zfs-* timer panels
+        "processes" # node_processes_* — "Processes Detailed States", "PIDs", "Threads"
+      ] ++ cfg.node.extraCollectors;
+
+      extraFlags = [
+        "--collector.textfile.directory=${cfg.textfileDir}"
+        # node_systemd_unit_start_time_seconds, so the dashboards can show when the
+        # zfs-snapshot-* and zfs-scrub units last ran rather than only whether they
+        # failed.
+        "--collector.systemd.enable-start-time-metrics"
+      ];
+    };
+
+    # Upstream hardens the unit with ProtectHome=true, which makes /home look empty and
+    # blanks out the filesystem panels for any dataset mounted under it. read-only still
+    # allows statfs while keeping the sandbox.
+    systemd.services.prometheus-node-exporter = mkIf cfg.node.enable {
+      serviceConfig.ProtectHome = mkForce "read-only";
+    };
+
+    # ------------------------------------------------------------------------------
+    # SMART
+    # ------------------------------------------------------------------------------
+    services.prometheus.exporters.smartctl = mkIf cfg.smart.enable {
+      enable = true;
+      port = ports.smartctl;
+      inherit (cfg) listenAddress;
+      inherit (cfg.smart) devices maxInterval;
+    };
+
+    systemd.services.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorService {
+      description = "Collect SMART self-test history and drive identity";
+      execStart = "${lib.getExe collectorPkgs.smart-selftest} --output ${cfg.textfileDir}/smart-selftest.prom";
+      # Same shape as the smartctl exporter's own hardening: raw device access to talk
+      # to the disks, and nothing else.
+      extraServiceConfig = {
+        AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
+        CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
+        DevicePolicy = "closed";
+        DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ];
+      };
+    });
+
+    systemd.timers.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorTimer {
+      description = "Collect SMART self-test history periodically";
+      onBootSec = "3min";
+      onUnitActiveSec = cfg.smart.selfTestInterval;
+      accuracySec = "1min";
+      unit = "smart-selftest-collector.service";
+    });
+
+    # ------------------------------------------------------------------------------
+    # ZFS
+    # ------------------------------------------------------------------------------
+    systemd.services.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorService {
+      description = "Collect ZFS pool, dataset and snapshot metrics";
+      execStart = lib.escapeShellArgs ([
+        (lib.getExe collectorPkgs.zfs-metrics)
+        "--output"
+        "${cfg.textfileDir}/zfs.prom"
+      ] ++ retentionArgs);
+      # CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated on it;
+      # everything else is dropped.
+      extraServiceConfig = {
+        CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ];
+        SystemCallFilter = [ "@system-service" ];
+      };
+    });
+
+    systemd.timers.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorTimer {
+      description = "Collect ZFS metrics periodically";
+      onBootSec = "1min";
+      onUnitActiveSec = cfg.zfs.interval;
+      accuracySec = "5s";
+      unit = "zfs-textfile-collector.service";
+    });
+
+    # An earlier version of the ZFS collector wrote zpool.prom. Nothing removes a stale
+    # textfile once its writer is gone, and node_exporter will happily keep serving
+    # those frozen metrics forever, so delete it on activation.
+    systemd.tmpfiles.rules = mkIf cfg.zfs.enable [
+      "r ${cfg.textfileDir}/zpool.prom"
+    ];
+
+    # ------------------------------------------------------------------------------
+    # Journal shipping
+    # ------------------------------------------------------------------------------
+    services.alloy.enable = mkIf cfg.logs.enable true;
+
+    environment.etc."alloy/config.alloy" = mkIf cfg.logs.enable {
+      text = ''
+        loki.source.journal "journal" {
+          forward_to    = [loki.write.default.receiver]
+          relabel_rules = loki.relabel.journal.rules
+          max_age       = "${cfg.logs.maxAge}"
+        }
+
+        loki.relabel "journal" {
+          forward_to = []
+          rule {
+            source_labels = ["__journal__systemd_unit"]
+            target_label  = "unit"
+          }
+          rule {
+            source_labels = ["__journal__hostname"]
+            target_label  = "host"
+          }
+          rule {
+            source_labels = ["__journal_priority_keyword"]
+            target_label  = "level"
+          }
+        }
+
+        loki.write "default" {
+          endpoint {
+            url = "${cfg.logs.lokiUrl}/loki/api/v1/push"
+          }
+        }
+      '';
+    };
+
+    # ------------------------------------------------------------------------------
+    # Firewall
+    # ------------------------------------------------------------------------------
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall (
+      optionals cfg.node.enable [ ports.node ]
+      ++ optionals cfg.smart.enable [ ports.smartctl ]
+    );
+  };
+}