split into a flake with separable collectors/server modules

Replaces the single lgtm.nix with a flake exposing two NixOS modules under
services.telemetry, so the web services can move into a container declared in
the consuming config rather than one hardcoded here.

  modules/collectors.nix  host-side: node + smartctl exporters, the ZFS and
                          SMART textfile collector timers, Alloy journal
                          shipping. Must run on the host -- reads /proc, /sys,
                          raw block devices and ZFS ioctls.
  modules/server.nix      Prometheus, Loki, Grafana, optional Caddy.
  modules/common.nix      ports, shared by both halves.
  modules/dashboards.nix  jq fixup and grafana.com fetches.
  modules/packages.nix    the two collectors as ordinary packages.

Modules are exported as paths, not functions, so they evaluate with the
consumer's nixpkgs; the flake's own input only affects packages/checks/devShells.

Multi-host support. collectorHosts is a list, both scrape jobs relabel
__address__ into a `host` label, and all 133 queries in the two generated
dashboards are scoped to host=~"$host" behind a single-select Host picker.
`instance` could not serve as that key: one machine is scraped twice on
different ports (textfile metrics on :9100, smartctl_* on :9633) and the
recording rules join across that boundary.

Grafana's secret key becomes a plain secretKeyFile path wired through
LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No
sops-nix input.

nix flake check runs promtool over the rules, diffs the committed dashboard
JSON against what the generators produce, and evaluates both modules.

Also fixes smart-selftest emitting a placeholder identity line when smartctl
cannot open a device. smartctl exits 0 with well-formed but empty JSON in that
case, so "SSD - unknown model - no serial" was reaching a dashboard row title;
it now emits nothing and lets the smartctl:ident fallback apply.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ

6 files changed, 967 insertions, 0 deletions

diff --git a/modules/collectors.nix b/modules/collectors.nix
new file mode 100644
index 0000000..6f328ee
--- /dev/null
+++ b/modules/collectors.nix
@@ -0,0 +1,368 @@
+# Host-side telemetry sources.
+#
+# Everything here has to run on the host, not in a container: it reads host hardware and
+# kernel state -- /proc, /sys, SMART over raw block devices, ZFS ioctls -- that a
+# container has no view of. The half that can be containerised is modules/server.nix.
+{ config, lib, pkgs, ... }:
+
+let
+  inherit (lib) mkIf mkOption mkEnableOption mkForce types optionals concatMap;
+
+  cfg = config.services.telemetry.collectors;
+  ports = config.services.telemetry.ports;
+
+  collectorPkgs = pkgs.callPackages ./packages.nix {
+    # Match the userland to the running kernel module.
+    zfs = config.boot.zfs.package;
+  };
+
+  # services.zfs.autoSnapshot's per-label keep counts, handed to the collector so it can
+  # report how far each label is from its target rather than just how many snapshots
+  # exist. Silently empty when auto-snapshotting is off, which the collector handles.
+  snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ];
+
+  retentionArgs = optionals config.services.zfs.autoSnapshot.enable (
+    concatMap
+      (label: [
+        "--retention"
+        "${label}=${toString config.services.zfs.autoSnapshot.${label}}"
+      ])
+      snapshotLabels
+  );
+
+  # Hardening common to both collectors. Each writes exactly one file into the textfile
+  # directory and talks to nothing over the network.
+  hardening = {
+    LockPersonality = true;
+    MemoryDenyWriteExecute = true;
+    NoNewPrivileges = true;
+    PrivateNetwork = true;
+    PrivateTmp = true;
+    ProtectControlGroups = true;
+    ProtectHome = true;
+    ProtectHostname = true;
+    ProtectKernelLogs = true;
+    ProtectKernelModules = true;
+    ProtectSystem = "strict";
+    RestrictAddressFamilies = [ "AF_UNIX" ];
+    RestrictNamespaces = true;
+    RestrictRealtime = true;
+    RestrictSUIDSGID = true;
+    SystemCallArchitectures = "native";
+  };
+
+  mkCollectorService = { description, execStart, extraServiceConfig ? { } }: {
+    inherit description;
+    serviceConfig = hardening // {
+      Type = "oneshot";
+      ExecStart = execStart;
+      User = "root";
+      StateDirectory = "prometheus-node-exporter-textfile";
+      StateDirectoryMode = "0755";
+      # node_exporter runs as a different user and has to be able to read what we write.
+      UMask = "0022";
+      TimeoutStartSec = "180s";
+    } // extraServiceConfig;
+  };
+
+  mkCollectorTimer = { description, onBootSec, onUnitActiveSec, accuracySec, unit }: {
+    inherit description;
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnBootSec = onBootSec;
+      OnUnitActiveSec = onUnitActiveSec;
+      AccuracySec = accuracySec;
+      Unit = unit;
+    };
+  };
+in
+{
+  imports = [ ./common.nix ];
+
+  options.services.telemetry.collectors = {
+    enable = mkEnableOption "host-side telemetry collectors (node exporter, SMART, ZFS)";
+
+    listenAddress = mkOption {
+      type = types.str;
+      default = "0.0.0.0";
+      description = ''
+        Address the exporters bind to. The default is deliberately permissive because
+        the whole point of splitting the halves is that the server scrapes these from
+        somewhere else -- a container on a bridge, or another machine. Narrow it if the
+        server runs on this host.
+      '';
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Open the node and smartctl exporter ports. Needed whenever the server is not on
+        this host's loopback, which includes a container with `privateNetwork`.
+      '';
+    };
+
+    textfileDir = mkOption {
+      type = types.path;
+      default = "/var/lib/prometheus-node-exporter-textfile";
+      description = ''
+        Directory the ZFS and SMART collectors write `.prom` files into, and that
+        node_exporter's textfile collector reads. Changing this also changes the
+        StateDirectory of both collector units.
+      '';
+    };
+
+    node = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run the Prometheus node exporter.";
+      };
+
+      extraCollectors = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        example = [ "interrupts" ];
+        description = ''
+          Additional node_exporter collectors, on top of the systemd, processes and
+          textfile collectors this module enables.
+
+          Note this *adds* to node_exporter's defaults -- cpu, diskstats, filesystem,
+          meminfo, zfs, hwmon and pressure are already on, so listing those does
+          nothing. `interrupts` is the notable opt-in: it fills the Node Exporter Full
+          "IRQ Detail" panel but emits one series per (cpu, irq) pair, which is
+          thousands of series on a many-core box.
+        '';
+      };
+    };
+
+    smart = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Run the smartctl exporter and the self-test history collector.
+        '';
+      };
+
+      devices = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        example = [ "/dev/sda" "/dev/nvme0" ];
+        description = ''
+          Devices to poll. Empty means auto-discovery via `smartctl --scan`, which is
+          correct for direct-attached AHCI/HBA disks. Behind a RAID controller you
+          generally have to list devices explicitly with their `-d` type.
+        '';
+      };
+
+      maxInterval = mkOption {
+        type = types.str;
+        default = "60s";
+        description = ''
+          Floor on how often the exporter actually shells out to smartctl, independent
+          of how often Prometheus scrapes it. SMART data moves slowly and polling can
+          spin up idle disks.
+        '';
+      };
+
+      selfTestInterval = mkOption {
+        type = types.str;
+        default = "15min";
+        description = ''
+          How often to re-read the self-test logs. The data only changes when a test
+          finishes, so this can be far slower than the exporter's own polling.
+        '';
+      };
+    };
+
+    zfs = {
+      enable = mkOption {
+        type = types.bool;
+        default = config.boot.zfs.enabled;
+        defaultText = lib.literalExpression "config.boot.zfs.enabled";
+        description = ''
+          Collect pool capacity and health, vdev errors, scrub history, dataset
+          properties and snapshot retention from the zpool/zfs CLIs.
+        '';
+      };
+
+      interval = mkOption {
+        type = types.str;
+        default = "1min";
+        description = ''
+          How often to re-read pool and dataset state. Listing every snapshot on a busy
+          pool is the slow part; if this starts overlapping its own runs, raise it.
+        '';
+      };
+    };
+
+    logs = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Ship the systemd journal to Loki with Grafana Alloy.";
+      };
+
+      lokiUrl = mkOption {
+        type = types.str;
+        default = "http://localhost:${toString ports.loki}";
+        defaultText = lib.literalExpression ''"http://localhost:''${toString config.services.telemetry.ports.loki}"'';
+        example = "http://metrics.lan:3100";
+        description = ''
+          Base URL of the Loki instance to push to. Point this at wherever
+          modules/server.nix runs; the `/loki/api/v1/push` path is appended.
+        '';
+      };
+
+      maxAge = mkOption {
+        type = types.str;
+        default = "12h";
+        description = ''
+          How far back into the journal to read on startup. Anything older than this is
+          skipped rather than replayed into Loki.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # ------------------------------------------------------------------------------
+    # node exporter
+    # ------------------------------------------------------------------------------
+    services.prometheus.exporters.node = mkIf cfg.node.enable {
+      enable = true;
+      port = ports.node;
+      inherit (cfg) listenAddress;
+
+      enabledCollectors = [
+        "systemd" # node_systemd_* — the "Systemd" row, and the zfs-* timer panels
+        "processes" # node_processes_* — "Processes Detailed States", "PIDs", "Threads"
+      ] ++ cfg.node.extraCollectors;
+
+      extraFlags = [
+        "--collector.textfile.directory=${cfg.textfileDir}"
+        # node_systemd_unit_start_time_seconds, so the dashboards can show when the
+        # zfs-snapshot-* and zfs-scrub units last ran rather than only whether they
+        # failed.
+        "--collector.systemd.enable-start-time-metrics"
+      ];
+    };
+
+    # Upstream hardens the unit with ProtectHome=true, which makes /home look empty and
+    # blanks out the filesystem panels for any dataset mounted under it. read-only still
+    # allows statfs while keeping the sandbox.
+    systemd.services.prometheus-node-exporter = mkIf cfg.node.enable {
+      serviceConfig.ProtectHome = mkForce "read-only";
+    };
+
+    # ------------------------------------------------------------------------------
+    # SMART
+    # ------------------------------------------------------------------------------
+    services.prometheus.exporters.smartctl = mkIf cfg.smart.enable {
+      enable = true;
+      port = ports.smartctl;
+      inherit (cfg) listenAddress;
+      inherit (cfg.smart) devices maxInterval;
+    };
+
+    systemd.services.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorService {
+      description = "Collect SMART self-test history and drive identity";
+      execStart = "${lib.getExe collectorPkgs.smart-selftest} --output ${cfg.textfileDir}/smart-selftest.prom";
+      # Same shape as the smartctl exporter's own hardening: raw device access to talk
+      # to the disks, and nothing else.
+      extraServiceConfig = {
+        AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
+        CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
+        DevicePolicy = "closed";
+        DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ];
+      };
+    });
+
+    systemd.timers.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorTimer {
+      description = "Collect SMART self-test history periodically";
+      onBootSec = "3min";
+      onUnitActiveSec = cfg.smart.selfTestInterval;
+      accuracySec = "1min";
+      unit = "smart-selftest-collector.service";
+    });
+
+    # ------------------------------------------------------------------------------
+    # ZFS
+    # ------------------------------------------------------------------------------
+    systemd.services.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorService {
+      description = "Collect ZFS pool, dataset and snapshot metrics";
+      execStart = lib.escapeShellArgs ([
+        (lib.getExe collectorPkgs.zfs-metrics)
+        "--output"
+        "${cfg.textfileDir}/zfs.prom"
+      ] ++ retentionArgs);
+      # CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated on it;
+      # everything else is dropped.
+      extraServiceConfig = {
+        CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ];
+        SystemCallFilter = [ "@system-service" ];
+      };
+    });
+
+    systemd.timers.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorTimer {
+      description = "Collect ZFS metrics periodically";
+      onBootSec = "1min";
+      onUnitActiveSec = cfg.zfs.interval;
+      accuracySec = "5s";
+      unit = "zfs-textfile-collector.service";
+    });
+
+    # An earlier version of the ZFS collector wrote zpool.prom. Nothing removes a stale
+    # textfile once its writer is gone, and node_exporter will happily keep serving
+    # those frozen metrics forever, so delete it on activation.
+    systemd.tmpfiles.rules = mkIf cfg.zfs.enable [
+      "r ${cfg.textfileDir}/zpool.prom"
+    ];
+
+    # ------------------------------------------------------------------------------
+    # Journal shipping
+    # ------------------------------------------------------------------------------
+    services.alloy.enable = mkIf cfg.logs.enable true;
+
+    environment.etc."alloy/config.alloy" = mkIf cfg.logs.enable {
+      text = ''
+        loki.source.journal "journal" {
+          forward_to    = [loki.write.default.receiver]
+          relabel_rules = loki.relabel.journal.rules
+          max_age       = "${cfg.logs.maxAge}"
+        }
+
+        loki.relabel "journal" {
+          forward_to = []
+          rule {
+            source_labels = ["__journal__systemd_unit"]
+            target_label  = "unit"
+          }
+          rule {
+            source_labels = ["__journal__hostname"]
+            target_label  = "host"
+          }
+          rule {
+            source_labels = ["__journal_priority_keyword"]
+            target_label  = "level"
+          }
+        }
+
+        loki.write "default" {
+          endpoint {
+            url = "${cfg.logs.lokiUrl}/loki/api/v1/push"
+          }
+        }
+      '';
+    };
+
+    # ------------------------------------------------------------------------------
+    # Firewall
+    # ------------------------------------------------------------------------------
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall (
+      optionals cfg.node.enable [ ports.node ]
+      ++ optionals cfg.smart.enable [ ports.smartctl ]
+    );
+  };
+}
diff --git a/modules/common.nix b/modules/common.nix
new file mode 100644
index 0000000..480aa62
--- /dev/null
+++ b/modules/common.nix
@@ -0,0 +1,46 @@
+# Options both halves need to agree on.
+#
+# The collectors bind these ports; the server scrapes them and pushes to them. When the
+# two halves live in separate NixOS configurations (host + container) there is no shared
+# `config` to read, so each side sets its own copy and it is on you to keep them equal —
+# which is why they are gathered in one small file rather than scattered.
+#
+# Imported by both modules/collectors.nix and modules/server.nix. Importing the same
+# path twice is harmless: the module system deduplicates imports by path, so a
+# configuration that uses both halves does not end up declaring these twice.
+{ lib, ... }:
+
+let
+  inherit (lib) mkOption types;
+
+  portOption = default: description: mkOption {
+    inherit default description;
+    type = types.port;
+  };
+in
+{
+  options.services.telemetry.ports = {
+    node = portOption 9100 ''
+      Port the Prometheus node exporter listens on. Also where the server's `node`
+      scrape job points.
+    '';
+
+    smartctl = portOption 9633 ''
+      Port the smartctl exporter listens on. Also where the server's `smartctl` scrape
+      job points.
+    '';
+
+    loki = portOption 3100 ''
+      Port Loki listens on. Also where the collectors' log shipper pushes to.
+    '';
+
+    prometheus = portOption 9090 ''
+      Port Prometheus listens on.
+    '';
+
+    grafana = portOption 3000 ''
+      Port Grafana listens on. If the reverse proxy is enabled this stays internal and
+      the proxy is what you connect to.
+    '';
+  };
+}
diff --git a/modules/dashboards.nix b/modules/dashboards.nix
new file mode 100644
index 0000000..125e3e2
--- /dev/null
+++ b/modules/dashboards.nix
@@ -0,0 +1,75 @@
+# Grafana dashboards, built into a form file-provisioning can actually use.
+#
+# Returns an attrset of { name = derivation; }, where each derivation is a single
+# dashboard JSON file. modules/server.nix drops these into /etc/grafana-dashboards.
+{ lib, runCommand, fetchurl, jq, dsUid }:
+
+let
+  # Dashboards exported from grafana.com carry an `__inputs` block and reference their
+  # datasource as "''${DS_PROMETHEUS}". That placeholder is only substituted by Grafana's
+  # interactive *import* flow -- file provisioning never runs it, so the uid stays
+  # literal and every panel fails to resolve a datasource. Substitute at build time.
+  #
+  # A dashboard-level `datasource`-type template variable (lowercase, e.g.
+  # "''${ds_prometheus}") is a different mechanism and resolves fine on its own; this
+  # only touches names declared in __inputs.
+  fixupProgram = ''
+    def dsuid($pluginId):
+      if   $pluginId == "prometheus" then $prom
+      elif $pluginId == "loki"       then $loki
+      else "" end;
+
+    def subst($name; $uid):
+      walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end);
+
+    reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId)))
+    | del(.__inputs, .__requires)
+    # Provisioned dashboards must not carry an id, or Grafana rejects them on collision
+    # with whatever already occupies that id in its database.
+    | .id = null
+  '';
+
+  fixup = name: src: runCommand "grafana-dashboard-${name}.json"
+    { nativeBuildInputs = [ jq ]; }
+    ''
+      jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \
+         --arg loki ${lib.escapeShellArg dsUid.loki} \
+         ${lib.escapeShellArg fixupProgram} < ${src} > "$out"
+    '';
+
+  fromGrafanaCom = { name, id, rev, hash }:
+    fixup name (fetchurl {
+      url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download";
+      inherit hash;
+    });
+in
+{
+  # https://grafana.com/grafana/dashboards/1860
+  node-exporter-full = fromGrafanaCom {
+    name = "node-exporter-full";
+    id = 1860;
+    rev = 45;
+    hash = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY=";
+  };
+
+  # https://grafana.com/grafana/dashboards/22604
+  smartctl-exporter = fromGrafanaCom {
+    name = "smartctl-exporter";
+    id = 22604;
+    rev = 2;
+    hash = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88=";
+  };
+
+  # Dashboard 15362 ("ZFS Pool Metrics") used to fill this slot. It is an *InfluxDB*
+  # dashboard -- its queries are InfluxQL against a `zpool_stats` measurement -- so it
+  # can never render against Prometheus, however it is provisioned. This is a
+  # Prometheus-native replacement built on the metrics we actually export: node
+  # exporter's zfs collector plus the ZFS textfile collector.
+  # Generated by dashboards/gen_zfs.py.
+  zfs-pools = fixup "zfs-pools" ../dashboards/zfs-pools.json;
+
+  # CrystalDiskInfo-style per-drive SMART view. Generated by dashboards/gen_smart.py.
+  # The Good/Caution/Bad verdict and the per-drive row titles both come from the
+  # recording rules in rules/smartctl.yml, so this one needs those loaded to be useful.
+  drive-health = fixup "drive-health" ../dashboards/smart.json;
+}
diff --git a/modules/default.nix b/modules/default.nix
new file mode 100644
index 0000000..07aa854
--- /dev/null
+++ b/modules/default.nix
@@ -0,0 +1,13 @@
+# Both halves at once, for running collectors and the web services on one machine.
+#
+# Importing this is equivalent to importing collectors.nix and server.nix together. If
+# you are splitting them -- collectors on the host, server in a container -- import the
+# two modules separately instead; each is self-contained.
+{ ... }:
+
+{
+  imports = [
+    ./collectors.nix
+    ./server.nix
+  ];
+}
diff --git a/modules/packages.nix b/modules/packages.nix
new file mode 100644
index 0000000..5e76f10
--- /dev/null
+++ b/modules/packages.nix
@@ -0,0 +1,45 @@
+# The two textfile collectors, as ordinary packages.
+#
+# They are wrappers rather than inline `writeShellApplication` bodies in the module so
+# that the Python is runnable by hand -- `zfs-metrics --output /tmp/x.prom` behaves
+# exactly as the timer does, which is how every one of these metrics was validated.
+# Neither wrapper hardcodes an output path; the unit passes it.
+{ lib, writeShellApplication, python3, smartmontools, zfs }:
+
+{
+  # Pool capacity and health, vdev errors, scrub history, dataset properties and
+  # snapshot retention -- none of which node_exporter's built-in zfs collector reports,
+  # because it reads /proc/spl/kstat/zfs and that only carries ARC statistics and objset
+  # counters. OpenZFS 2.2+ also dropped the per-pool `io` kstat, so node_zfs_zpool_* is
+  # largely empty. This shells out to the zpool/zfs CLIs instead.
+  #
+  # runtimeInputs is the whole reason for the wrapper: the collector must run against
+  # the same zfs userland as the running kernel module.
+  zfs-metrics = writeShellApplication {
+    name = "zfs-metrics";
+    runtimeInputs = [ zfs ];
+    text = ''
+      exec ${python3.interpreter} ${../collectors/zfs-metrics.py} "$@"
+    '';
+    meta = {
+      description = "Export ZFS pool, dataset and snapshot metrics in Prometheus format";
+      mainProgram = "zfs-metrics";
+    };
+  };
+
+  # smartctl_exporter reads only the entry *count* out of each drive's self-test log and
+  # discards the log table, so it cannot say when a drive last ran a short or extended
+  # test or whether it passed. It also has no way to render a drive's identity as a
+  # display string, which the dashboards need for row titles. Both come from here.
+  smart-selftest = writeShellApplication {
+    name = "smart-selftest";
+    runtimeInputs = [ smartmontools ];
+    text = ''
+      exec ${python3.interpreter} ${../collectors/smart-selftest.py} "$@"
+    '';
+    meta = {
+      description = "Export SMART self-test history and drive identity in Prometheus format";
+      mainProgram = "smart-selftest";
+    };
+  };
+}
diff --git a/modules/server.nix b/modules/server.nix
new file mode 100644
index 0000000..1996f5e
--- /dev/null
+++ b/modules/server.nix
@@ -0,0 +1,420 @@
+# The web half: Prometheus, Loki, Grafana and an optional reverse proxy.
+#
+# Nothing here reads host hardware, so this is the part you can put in a container. It
+# reaches the collectors over the network -- see `collectorHosts`.
+{ config, lib, pkgs, ... }:
+
+let
+  inherit (lib) mkIf mkOption mkEnableOption mkDefault types optionals filterAttrs mapAttrs' nameValuePair;
+
+  cfg = config.services.telemetry.server;
+  ports = config.services.telemetry.ports;
+
+  # Fixed, because the generated dashboards reference them literally. dashboards/panels.py
+  # emits {"type": "prometheus", "uid": "prometheus"} on every panel, and the jq fixup in
+  # dashboards.nix rewrites grafana.com's placeholders to the same values. Changing these
+  # would silently break every panel, so they are not options.
+  dsUid = { prometheus = "prometheus"; loki = "loki"; };
+
+  # A `host` label carrying the machine name with the port stripped off.
+  #
+  # `instance` cannot do this job. The same machine is scraped twice on different ports
+  # -- the node exporter on :9100 serves the textfile metrics (zfs_*, smart_selftest_*,
+  # smart_drive_label) while smartctl_* comes from its own exporter on :9633 -- so one
+  # machine has two `instance` values, and the recording rules join straight across that
+  # boundary. `host` is equal on both, which is what makes the dashboards' host picker
+  # and those joins work at the same time.
+  #
+  # Note the regex is fully anchored by Prometheus, and only handles host:port. A
+  # bracketed IPv6 literal needs an explicit label via `extraScrapeConfigs` instead.
+  hostLabelRelabel = [{
+    source_labels = [ "__address__" ];
+    regex = "([^:]+)(?::[0-9]+)?";
+    target_label = "host";
+    replacement = "$1";
+  }];
+
+  builtinDashboards = pkgs.callPackage ./dashboards.nix { inherit dsUid; };
+
+  selected = filterAttrs (name: _: cfg.grafana.dashboards.${name} or false) builtinDashboards;
+
+  dashboards = selected // cfg.grafana.extraDashboards;
+
+  dashboardOption = description: mkOption {
+    inherit description;
+    type = types.bool;
+    default = true;
+  };
+in
+{
+  imports = [ ./common.nix ];
+
+  options.services.telemetry.server = {
+    enable = mkEnableOption "the Prometheus/Loki/Grafana server";
+
+    collectorHosts = mkOption {
+      type = types.listOf types.str;
+      default = [ "localhost" ];
+      example = [ "poweredge" "optiplex" ];
+      description = ''
+        Machines running modules/collectors.nix. Each is scraped on both the node and
+        smartctl ports from {option}`services.telemetry.ports`, so this is normally the
+        only thing on the server side that has to know about the hosts.
+
+        Each entry also becomes the value of the `host` label on everything scraped
+        from that machine, which is what the dashboards' host picker selects on. Use
+        the same spelling you would type in a browser; if you list an IP address, that
+        is what the picker will show.
+
+        Set {option}`services.telemetry.server.prometheus.nodeTargets` or
+        {option}`services.telemetry.server.prometheus.smartctlTargets` to take over
+        target generation entirely.
+      '';
+    };
+
+    listenAddress = mkOption {
+      type = types.str;
+      default = "0.0.0.0";
+      description = ''
+        Address Loki, Prometheus and Grafana bind to. Loki has to be reachable from
+        every host running the collectors, so this defaults to all interfaces.
+      '';
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Open the ports of whichever services are enabled: Loki (the collectors push to
+        it), Prometheus (useful for checking scrape health directly), and either the
+        proxy or Grafana.
+      '';
+    };
+
+    prometheus = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run Prometheus.";
+      };
+
+      scrapeInterval = mkOption {
+        type = types.str;
+        default = "15s";
+        description = ''
+          Global scrape interval, and the value handed to Grafana's Prometheus
+          datasource as `timeInterval`.
+
+          These two MUST agree. Grafana derives `$__rate_interval` from `timeInterval`,
+          and if it guesses wrong then `rate()` windows are mis-sized and every
+          rate-based panel intermittently renders "No data" -- which is exactly what
+          Prometheus' own 1m default does to the Node Exporter Full dashboard. Setting
+          it here sets both.
+        '';
+      };
+
+      nodeTargets = mkOption {
+        type = types.listOf types.str;
+        default = map (host: "${host}:${toString ports.node}") cfg.collectorHosts;
+        defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the node exporter port";
+        description = "Targets for the `node` scrape job.";
+      };
+
+      smartctlTargets = mkOption {
+        type = types.listOf types.str;
+        default = map (host: "${host}:${toString ports.smartctl}") cfg.collectorHosts;
+        defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the smartctl exporter port";
+        description = "Targets for the `smartctl` scrape job. Empty disables the job.";
+      };
+
+      smartctlScrapeInterval = mkOption {
+        type = types.str;
+        default = "60s";
+        description = ''
+          Scrape interval for the smartctl job, which opts out of the global one:
+          querying SMART is slow and the data barely moves. Safe because the drive
+          dashboards use no `rate()` queries at all.
+        '';
+      };
+
+      ruleFiles = mkOption {
+        type = types.listOf types.path;
+        default = [ ../rules/smartctl.yml ];
+        defaultText = lib.literalExpression "[ ../rules/smartctl.yml ]";
+        description = ''
+          Recording rule files. The default computes the per-drive Good/Caution/Bad
+          verdict and the row-title labels that the drive health dashboard displays, so
+          drop it only if you also drop that dashboard.
+
+          {option}`services.prometheus.checkConfig` runs promtool over these at build
+          time, so a broken rule fails the build rather than the service.
+        '';
+      };
+
+      extraScrapeConfigs = mkOption {
+        type = types.listOf types.attrs;
+        default = [ ];
+        description = "Additional scrape jobs, appended to the generated ones.";
+      };
+
+      retentionTime = mkOption {
+        type = types.str;
+        default = "90d";
+        description = "How long to keep samples.";
+      };
+    };
+
+    loki = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run Loki to receive journal logs from the collectors.";
+      };
+
+      dataDir = mkOption {
+        type = types.path;
+        default = "/var/lib/loki";
+        description = "Where Loki stores its chunks and index.";
+      };
+    };
+
+    grafana = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run Grafana, provisioned with datasources and dashboards.";
+      };
+
+      secretKeyFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        example = lib.literalExpression "config.sops.secrets.grafana-secret-key.path";
+        description = ''
+          File holding Grafana's `security.secret_key`. Loaded through systemd's
+          `LoadCredential`, so the file only has to be readable by root at start time
+          and never lands in the Nix store.
+
+          Deliberately a plain path rather than a sops-nix integration: wire it up to
+          whatever secret manager you already use.
+        '';
+      };
+
+      dashboards = {
+        node-exporter-full = dashboardOption "Provision grafana.com dashboard 1860 (Node Exporter Full).";
+        smartctl-exporter = dashboardOption "Provision grafana.com dashboard 22604 (Smartctl Exporter).";
+        zfs-pools = dashboardOption "Provision the ZFS pools dashboard. Needs the ZFS collector.";
+        drive-health = dashboardOption "Provision the CrystalDiskInfo-style drive health dashboard. Needs the SMART collector and the smartctl recording rules.";
+      };
+
+      extraDashboards = mkOption {
+        type = types.attrsOf types.path;
+        default = { };
+        example = lib.literalExpression ''{ my-dashboard = ./my-dashboard.json; }'';
+        description = ''
+          Extra dashboards to provision, as { name = path-to-json; }. These are passed
+          through unmodified, so they must already reference the datasource UIDs
+          "prometheus" and "loki".
+        '';
+      };
+    };
+
+    proxy = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Run Caddy in front of Grafana. Off by default because a reverse proxy is
+          usually something you already have; turn it on to get Grafana on port 80
+          without exposing its own port.
+        '';
+      };
+
+      virtualHost = mkOption {
+        type = types.str;
+        default = ":80";
+        example = "metrics.example.com";
+        description = ''
+          Caddy virtual host. A bare hostname makes Caddy provision a certificate
+          automatically; ":80" serves plain HTTP on every name.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # Grafana uses this key to encrypt datasource credentials in its database, and
+    # nixpkgs stopped shipping a default in 26.05. Caught here rather than left to
+    # Grafana's own assertion, which names an option this module manages for you.
+    assertions = [{
+      assertion = !cfg.grafana.enable || cfg.grafana.secretKeyFile != null;
+      message = ''
+        services.telemetry.server.grafana.secretKeyFile must be set when Grafana is
+        enabled. Generate a key (`openssl rand -base64 32`), store it with whatever
+        secret manager you use, and point this option at the resulting file.
+      '';
+    }];
+
+    # --------------------------------------------------------------------------------
+    # Loki
+    # --------------------------------------------------------------------------------
+    services.loki = mkIf cfg.loki.enable {
+      enable = true;
+      configuration = {
+        auth_enabled = false;
+
+        server = {
+          http_listen_port = ports.loki;
+          # Explicit: the collectors have to reach this from another host or container.
+          http_listen_address = cfg.listenAddress;
+        };
+
+        common = {
+          path_prefix = cfg.loki.dataDir;
+          storage.filesystem = {
+            chunks_directory = "${cfg.loki.dataDir}/chunks";
+            rules_directory = "${cfg.loki.dataDir}/rules";
+          };
+          ring = {
+            instance_addr = "127.0.0.1";
+            kvstore.store = "inmemory";
+          };
+          replication_factor = 1;
+        };
+
+        schema_config.configs = [{
+          from = "2024-01-01";
+          store = "tsdb";
+          object_store = "filesystem";
+          schema = "v13";
+          index = { prefix = "index_"; period = "24h"; };
+        }];
+      };
+    };
+
+    # --------------------------------------------------------------------------------
+    # Prometheus
+    # --------------------------------------------------------------------------------
+    services.prometheus = mkIf cfg.prometheus.enable {
+      enable = true;
+      port = ports.prometheus;
+      listenAddress = cfg.listenAddress;
+      retentionTime = cfg.prometheus.retentionTime;
+
+      globalConfig = {
+        scrape_interval = cfg.prometheus.scrapeInterval;
+        evaluation_interval = cfg.prometheus.scrapeInterval;
+        scrape_timeout = "10s";
+      };
+
+      inherit (cfg.prometheus) ruleFiles;
+
+      scrapeConfigs =
+        optionals (cfg.prometheus.nodeTargets != [ ]) [{
+          job_name = "node";
+          relabel_configs = hostLabelRelabel;
+          static_configs = [{ targets = cfg.prometheus.nodeTargets; }];
+        }]
+        ++ optionals (cfg.prometheus.smartctlTargets != [ ]) [{
+          job_name = "smartctl";
+          scrape_interval = cfg.prometheus.smartctlScrapeInterval;
+          scrape_timeout = "30s";
+          relabel_configs = hostLabelRelabel;
+          static_configs = [{ targets = cfg.prometheus.smartctlTargets; }];
+        }]
+        ++ cfg.prometheus.extraScrapeConfigs;
+    };
+
+    # --------------------------------------------------------------------------------
+    # Grafana
+    # --------------------------------------------------------------------------------
+    services.grafana = mkIf cfg.grafana.enable {
+      enable = true;
+
+      settings = {
+        server = {
+          http_addr = mkDefault cfg.listenAddress;
+          http_port = mkDefault ports.grafana;
+        };
+      }
+      # A file provider rather than a literal, so the key is never written into a
+      # world-readable store path. The credential is mounted by LoadCredential below.
+      // lib.optionalAttrs (cfg.grafana.secretKeyFile != null) {
+        security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}";
+      };
+
+      provision = {
+        enable = true;
+
+        dashboards.settings.providers = [{
+          name = "provisioned";
+          disableDeletion = true;
+          options = {
+            path = "/etc/grafana-dashboards";
+            foldersFromFilesStructure = true;
+          };
+        }];
+
+        # Grafana cannot change the UID of an existing datasource in place. Its
+        # provisioner finds the datasource by name, then calls UpdateDataSource, which
+        # re-fetches using `id AND uid` -- with a new uid that row does not match, so it
+        # fails with "data source not found" and Grafana refuses to start. Deleting
+        # first turns the upsert into a plain insert; the provisioner runs deletes
+        # before inserts for exactly this case and suppresses the deletion event because
+        # the datasource is about to be re-created. Safe permanently: both datasources
+        # are fully declarative and dashboards bind to them by UID, not database ID.
+        datasources.settings.deleteDatasources =
+          [{ name = "Prometheus"; orgId = 1; }]
+          ++ optionals cfg.loki.enable [{ name = "Loki"; orgId = 1; }];
+
+        datasources.settings.datasources =
+          optionals cfg.prometheus.enable [{
+            name = "Prometheus";
+            type = "prometheus";
+            uid = dsUid.prometheus;
+            url = "http://127.0.0.1:${toString ports.prometheus}";
+            isDefault = true;
+            jsonData = {
+              # What $__rate_interval is derived from. Leaving it unset makes Grafana
+              # assume 15s and mis-size every rate() window. See scrapeInterval above.
+              timeInterval = cfg.prometheus.scrapeInterval;
+              httpMethod = "POST";
+            };
+          }]
+          ++ optionals cfg.loki.enable [{
+            name = "Loki";
+            type = "loki";
+            uid = dsUid.loki;
+            url = "http://127.0.0.1:${toString ports.loki}";
+          }];
+      };
+    };
+
+    systemd.services.grafana = mkIf (cfg.grafana.enable && cfg.grafana.secretKeyFile != null) {
+      serviceConfig.LoadCredential = [ "secret_key:${cfg.grafana.secretKeyFile}" ];
+    };
+
+    environment.etc = mkIf cfg.grafana.enable (
+      mapAttrs' (name: src: nameValuePair "grafana-dashboards/${name}.json" { source = src; })
+        dashboards
+    );
+
+    # --------------------------------------------------------------------------------
+    # Reverse proxy
+    # --------------------------------------------------------------------------------
+    services.caddy = mkIf cfg.proxy.enable {
+      enable = true;
+      virtualHosts.${cfg.proxy.virtualHost}.extraConfig = ''
+        reverse_proxy localhost:${toString ports.grafana}
+      '';
+    };
+
+    # --------------------------------------------------------------------------------
+    # Firewall
+    # --------------------------------------------------------------------------------
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall (
+      optionals cfg.loki.enable [ ports.loki ]
+      ++ optionals cfg.prometheus.enable [ ports.prometheus ]
+      ++ (if cfg.proxy.enable then [ 80 443 ] else optionals cfg.grafana.enable [ ports.grafana ])
+    );
+  };
+}