split into a flake with separable collectors/server modules

Replaces the single lgtm.nix with a flake exposing two NixOS modules under
services.telemetry, so the web services can move into a container declared in
the consuming config rather than one hardcoded here.

  modules/collectors.nix  host-side: node + smartctl exporters, the ZFS and
                          SMART textfile collector timers, Alloy journal
                          shipping. Must run on the host -- reads /proc, /sys,
                          raw block devices and ZFS ioctls.
  modules/server.nix      Prometheus, Loki, Grafana, optional Caddy.
  modules/common.nix      ports, shared by both halves.
  modules/dashboards.nix  jq fixup and grafana.com fetches.
  modules/packages.nix    the two collectors as ordinary packages.

Modules are exported as paths, not functions, so they evaluate with the
consumer's nixpkgs; the flake's own input only affects packages/checks/devShells.

Multi-host support. collectorHosts is a list, both scrape jobs relabel
__address__ into a `host` label, and all 133 queries in the two generated
dashboards are scoped to host=~"$host" behind a single-select Host picker.
`instance` could not serve as that key: one machine is scraped twice on
different ports (textfile metrics on :9100, smartctl_* on :9633) and the
recording rules join across that boundary.

Grafana's secret key becomes a plain secretKeyFile path wired through
LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No
sops-nix input.

nix flake check runs promtool over the rules, diffs the committed dashboard
JSON against what the generators produce, and evaluates both modules.

Also fixes smart-selftest emitting a placeholder identity line when smartctl
cannot open a device. smartctl exits 0 with well-formed but empty JSON in that
case, so "SSD - unknown model - no serial" was reaching a dashboard row title;
it now emits nothing and lets the smartctl:ident fallback apply.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ

1 files changed, 420 insertions, 0 deletions

diff --git a/modules/server.nix b/modules/server.nix
new file mode 100644
index 0000000..1996f5e
--- /dev/null
+++ b/modules/server.nix
@@ -0,0 +1,420 @@
+# The web half: Prometheus, Loki, Grafana and an optional reverse proxy.
+#
+# Nothing here reads host hardware, so this is the part you can put in a container. It
+# reaches the collectors over the network -- see `collectorHosts`.
+{ config, lib, pkgs, ... }:
+
+let
+  inherit (lib) mkIf mkOption mkEnableOption mkDefault types optionals filterAttrs mapAttrs' nameValuePair;
+
+  cfg = config.services.telemetry.server;
+  ports = config.services.telemetry.ports;
+
+  # Fixed, because the generated dashboards reference them literally. dashboards/panels.py
+  # emits {"type": "prometheus", "uid": "prometheus"} on every panel, and the jq fixup in
+  # dashboards.nix rewrites grafana.com's placeholders to the same values. Changing these
+  # would silently break every panel, so they are not options.
+  dsUid = { prometheus = "prometheus"; loki = "loki"; };
+
+  # A `host` label carrying the machine name with the port stripped off.
+  #
+  # `instance` cannot do this job. The same machine is scraped twice on different ports
+  # -- the node exporter on :9100 serves the textfile metrics (zfs_*, smart_selftest_*,
+  # smart_drive_label) while smartctl_* comes from its own exporter on :9633 -- so one
+  # machine has two `instance` values, and the recording rules join straight across that
+  # boundary. `host` is equal on both, which is what makes the dashboards' host picker
+  # and those joins work at the same time.
+  #
+  # Note the regex is fully anchored by Prometheus, and only handles host:port. A
+  # bracketed IPv6 literal needs an explicit label via `extraScrapeConfigs` instead.
+  hostLabelRelabel = [{
+    source_labels = [ "__address__" ];
+    regex = "([^:]+)(?::[0-9]+)?";
+    target_label = "host";
+    replacement = "$1";
+  }];
+
+  builtinDashboards = pkgs.callPackage ./dashboards.nix { inherit dsUid; };
+
+  selected = filterAttrs (name: _: cfg.grafana.dashboards.${name} or false) builtinDashboards;
+
+  dashboards = selected // cfg.grafana.extraDashboards;
+
+  dashboardOption = description: mkOption {
+    inherit description;
+    type = types.bool;
+    default = true;
+  };
+in
+{
+  imports = [ ./common.nix ];
+
+  options.services.telemetry.server = {
+    enable = mkEnableOption "the Prometheus/Loki/Grafana server";
+
+    collectorHosts = mkOption {
+      type = types.listOf types.str;
+      default = [ "localhost" ];
+      example = [ "poweredge" "optiplex" ];
+      description = ''
+        Machines running modules/collectors.nix. Each is scraped on both the node and
+        smartctl ports from {option}`services.telemetry.ports`, so this is normally the
+        only thing on the server side that has to know about the hosts.
+
+        Each entry also becomes the value of the `host` label on everything scraped
+        from that machine, which is what the dashboards' host picker selects on. Use
+        the same spelling you would type in a browser; if you list an IP address, that
+        is what the picker will show.
+
+        Set {option}`services.telemetry.server.prometheus.nodeTargets` or
+        {option}`services.telemetry.server.prometheus.smartctlTargets` to take over
+        target generation entirely.
+      '';
+    };
+
+    listenAddress = mkOption {
+      type = types.str;
+      default = "0.0.0.0";
+      description = ''
+        Address Loki, Prometheus and Grafana bind to. Loki has to be reachable from
+        every host running the collectors, so this defaults to all interfaces.
+      '';
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Open the ports of whichever services are enabled: Loki (the collectors push to
+        it), Prometheus (useful for checking scrape health directly), and either the
+        proxy or Grafana.
+      '';
+    };
+
+    prometheus = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run Prometheus.";
+      };
+
+      scrapeInterval = mkOption {
+        type = types.str;
+        default = "15s";
+        description = ''
+          Global scrape interval, and the value handed to Grafana's Prometheus
+          datasource as `timeInterval`.
+
+          These two MUST agree. Grafana derives `$__rate_interval` from `timeInterval`,
+          and if it guesses wrong then `rate()` windows are mis-sized and every
+          rate-based panel intermittently renders "No data" -- which is exactly what
+          Prometheus' own 1m default does to the Node Exporter Full dashboard. Setting
+          it here sets both.
+        '';
+      };
+
+      nodeTargets = mkOption {
+        type = types.listOf types.str;
+        default = map (host: "${host}:${toString ports.node}") cfg.collectorHosts;
+        defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the node exporter port";
+        description = "Targets for the `node` scrape job.";
+      };
+
+      smartctlTargets = mkOption {
+        type = types.listOf types.str;
+        default = map (host: "${host}:${toString ports.smartctl}") cfg.collectorHosts;
+        defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the smartctl exporter port";
+        description = "Targets for the `smartctl` scrape job. Empty disables the job.";
+      };
+
+      smartctlScrapeInterval = mkOption {
+        type = types.str;
+        default = "60s";
+        description = ''
+          Scrape interval for the smartctl job, which opts out of the global one:
+          querying SMART is slow and the data barely moves. Safe because the drive
+          dashboards use no `rate()` queries at all.
+        '';
+      };
+
+      ruleFiles = mkOption {
+        type = types.listOf types.path;
+        default = [ ../rules/smartctl.yml ];
+        defaultText = lib.literalExpression "[ ../rules/smartctl.yml ]";
+        description = ''
+          Recording rule files. The default computes the per-drive Good/Caution/Bad
+          verdict and the row-title labels that the drive health dashboard displays, so
+          drop it only if you also drop that dashboard.
+
+          {option}`services.prometheus.checkConfig` runs promtool over these at build
+          time, so a broken rule fails the build rather than the service.
+        '';
+      };
+
+      extraScrapeConfigs = mkOption {
+        type = types.listOf types.attrs;
+        default = [ ];
+        description = "Additional scrape jobs, appended to the generated ones.";
+      };
+
+      retentionTime = mkOption {
+        type = types.str;
+        default = "90d";
+        description = "How long to keep samples.";
+      };
+    };
+
+    loki = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run Loki to receive journal logs from the collectors.";
+      };
+
+      dataDir = mkOption {
+        type = types.path;
+        default = "/var/lib/loki";
+        description = "Where Loki stores its chunks and index.";
+      };
+    };
+
+    grafana = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run Grafana, provisioned with datasources and dashboards.";
+      };
+
+      secretKeyFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        example = lib.literalExpression "config.sops.secrets.grafana-secret-key.path";
+        description = ''
+          File holding Grafana's `security.secret_key`. Loaded through systemd's
+          `LoadCredential`, so the file only has to be readable by root at start time
+          and never lands in the Nix store.
+
+          Deliberately a plain path rather than a sops-nix integration: wire it up to
+          whatever secret manager you already use.
+        '';
+      };
+
+      dashboards = {
+        node-exporter-full = dashboardOption "Provision grafana.com dashboard 1860 (Node Exporter Full).";
+        smartctl-exporter = dashboardOption "Provision grafana.com dashboard 22604 (Smartctl Exporter).";
+        zfs-pools = dashboardOption "Provision the ZFS pools dashboard. Needs the ZFS collector.";
+        drive-health = dashboardOption "Provision the CrystalDiskInfo-style drive health dashboard. Needs the SMART collector and the smartctl recording rules.";
+      };
+
+      extraDashboards = mkOption {
+        type = types.attrsOf types.path;
+        default = { };
+        example = lib.literalExpression ''{ my-dashboard = ./my-dashboard.json; }'';
+        description = ''
+          Extra dashboards to provision, as { name = path-to-json; }. These are passed
+          through unmodified, so they must already reference the datasource UIDs
+          "prometheus" and "loki".
+        '';
+      };
+    };
+
+    proxy = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Run Caddy in front of Grafana. Off by default because a reverse proxy is
+          usually something you already have; turn it on to get Grafana on port 80
+          without exposing its own port.
+        '';
+      };
+
+      virtualHost = mkOption {
+        type = types.str;
+        default = ":80";
+        example = "metrics.example.com";
+        description = ''
+          Caddy virtual host. A bare hostname makes Caddy provision a certificate
+          automatically; ":80" serves plain HTTP on every name.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # Grafana uses this key to encrypt datasource credentials in its database, and
+    # nixpkgs stopped shipping a default in 26.05. Caught here rather than left to
+    # Grafana's own assertion, which names an option this module manages for you.
+    assertions = [{
+      assertion = !cfg.grafana.enable || cfg.grafana.secretKeyFile != null;
+      message = ''
+        services.telemetry.server.grafana.secretKeyFile must be set when Grafana is
+        enabled. Generate a key (`openssl rand -base64 32`), store it with whatever
+        secret manager you use, and point this option at the resulting file.
+      '';
+    }];
+
+    # --------------------------------------------------------------------------------
+    # Loki
+    # --------------------------------------------------------------------------------
+    services.loki = mkIf cfg.loki.enable {
+      enable = true;
+      configuration = {
+        auth_enabled = false;
+
+        server = {
+          http_listen_port = ports.loki;
+          # Explicit: the collectors have to reach this from another host or container.
+          http_listen_address = cfg.listenAddress;
+        };
+
+        common = {
+          path_prefix = cfg.loki.dataDir;
+          storage.filesystem = {
+            chunks_directory = "${cfg.loki.dataDir}/chunks";
+            rules_directory = "${cfg.loki.dataDir}/rules";
+          };
+          ring = {
+            instance_addr = "127.0.0.1";
+            kvstore.store = "inmemory";
+          };
+          replication_factor = 1;
+        };
+
+        schema_config.configs = [{
+          from = "2024-01-01";
+          store = "tsdb";
+          object_store = "filesystem";
+          schema = "v13";
+          index = { prefix = "index_"; period = "24h"; };
+        }];
+      };
+    };
+
+    # --------------------------------------------------------------------------------
+    # Prometheus
+    # --------------------------------------------------------------------------------
+    services.prometheus = mkIf cfg.prometheus.enable {
+      enable = true;
+      port = ports.prometheus;
+      listenAddress = cfg.listenAddress;
+      retentionTime = cfg.prometheus.retentionTime;
+
+      globalConfig = {
+        scrape_interval = cfg.prometheus.scrapeInterval;
+        evaluation_interval = cfg.prometheus.scrapeInterval;
+        scrape_timeout = "10s";
+      };
+
+      inherit (cfg.prometheus) ruleFiles;
+
+      scrapeConfigs =
+        optionals (cfg.prometheus.nodeTargets != [ ]) [{
+          job_name = "node";
+          relabel_configs = hostLabelRelabel;
+          static_configs = [{ targets = cfg.prometheus.nodeTargets; }];
+        }]
+        ++ optionals (cfg.prometheus.smartctlTargets != [ ]) [{
+          job_name = "smartctl";
+          scrape_interval = cfg.prometheus.smartctlScrapeInterval;
+          scrape_timeout = "30s";
+          relabel_configs = hostLabelRelabel;
+          static_configs = [{ targets = cfg.prometheus.smartctlTargets; }];
+        }]
+        ++ cfg.prometheus.extraScrapeConfigs;
+    };
+
+    # --------------------------------------------------------------------------------
+    # Grafana
+    # --------------------------------------------------------------------------------
+    services.grafana = mkIf cfg.grafana.enable {
+      enable = true;
+
+      settings = {
+        server = {
+          http_addr = mkDefault cfg.listenAddress;
+          http_port = mkDefault ports.grafana;
+        };
+      }
+      # A file provider rather than a literal, so the key is never written into a
+      # world-readable store path. The credential is mounted by LoadCredential below.
+      // lib.optionalAttrs (cfg.grafana.secretKeyFile != null) {
+        security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}";
+      };
+
+      provision = {
+        enable = true;
+
+        dashboards.settings.providers = [{
+          name = "provisioned";
+          disableDeletion = true;
+          options = {
+            path = "/etc/grafana-dashboards";
+            foldersFromFilesStructure = true;
+          };
+        }];
+
+        # Grafana cannot change the UID of an existing datasource in place. Its
+        # provisioner finds the datasource by name, then calls UpdateDataSource, which
+        # re-fetches using `id AND uid` -- with a new uid that row does not match, so it
+        # fails with "data source not found" and Grafana refuses to start. Deleting
+        # first turns the upsert into a plain insert; the provisioner runs deletes
+        # before inserts for exactly this case and suppresses the deletion event because
+        # the datasource is about to be re-created. Safe permanently: both datasources
+        # are fully declarative and dashboards bind to them by UID, not database ID.
+        datasources.settings.deleteDatasources =
+          [{ name = "Prometheus"; orgId = 1; }]
+          ++ optionals cfg.loki.enable [{ name = "Loki"; orgId = 1; }];
+
+        datasources.settings.datasources =
+          optionals cfg.prometheus.enable [{
+            name = "Prometheus";
+            type = "prometheus";
+            uid = dsUid.prometheus;
+            url = "http://127.0.0.1:${toString ports.prometheus}";
+            isDefault = true;
+            jsonData = {
+              # What $__rate_interval is derived from. Leaving it unset makes Grafana
+              # assume 15s and mis-size every rate() window. See scrapeInterval above.
+              timeInterval = cfg.prometheus.scrapeInterval;
+              httpMethod = "POST";
+            };
+          }]
+          ++ optionals cfg.loki.enable [{
+            name = "Loki";
+            type = "loki";
+            uid = dsUid.loki;
+            url = "http://127.0.0.1:${toString ports.loki}";
+          }];
+      };
+    };
+
+    systemd.services.grafana = mkIf (cfg.grafana.enable && cfg.grafana.secretKeyFile != null) {
+      serviceConfig.LoadCredential = [ "secret_key:${cfg.grafana.secretKeyFile}" ];
+    };
+
+    environment.etc = mkIf cfg.grafana.enable (
+      mapAttrs' (name: src: nameValuePair "grafana-dashboards/${name}.json" { source = src; })
+        dashboards
+    );
+
+    # --------------------------------------------------------------------------------
+    # Reverse proxy
+    # --------------------------------------------------------------------------------
+    services.caddy = mkIf cfg.proxy.enable {
+      enable = true;
+      virtualHosts.${cfg.proxy.virtualHost}.extraConfig = ''
+        reverse_proxy localhost:${toString ports.grafana}
+      '';
+    };
+
+    # --------------------------------------------------------------------------------
+    # Firewall
+    # --------------------------------------------------------------------------------
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall (
+      optionals cfg.loki.enable [ ports.loki ]
+      ++ optionals cfg.prometheus.enable [ ports.prometheus ]
+      ++ (if cfg.proxy.enable then [ 80 443 ] else optionals cfg.grafana.enable [ ports.grafana ])
+    );
+  };
+}