split into a flake with separable collectors/server modules

Replaces the single lgtm.nix with a flake exposing two NixOS modules under
services.telemetry, so the web services can move into a container declared in
the consuming config rather than one hardcoded here.

  modules/collectors.nix  host-side: node + smartctl exporters, the ZFS and
                          SMART textfile collector timers, Alloy journal
                          shipping. Must run on the host -- reads /proc, /sys,
                          raw block devices and ZFS ioctls.
  modules/server.nix      Prometheus, Loki, Grafana, optional Caddy.
  modules/common.nix      ports, shared by both halves.
  modules/dashboards.nix  jq fixup and grafana.com fetches.
  modules/packages.nix    the two collectors as ordinary packages.

Modules are exported as paths, not functions, so they evaluate with the
consumer's nixpkgs; the flake's own input only affects packages/checks/devShells.

Multi-host support. collectorHosts is a list, both scrape jobs relabel
__address__ into a `host` label, and all 133 queries in the two generated
dashboards are scoped to host=~"$host" behind a single-select Host picker.
`instance` could not serve as that key: one machine is scraped twice on
different ports (textfile metrics on :9100, smartctl_* on :9633) and the
recording rules join across that boundary.

Grafana's secret key becomes a plain secretKeyFile path wired through
LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No
sops-nix input.

nix flake check runs promtool over the rules, diffs the committed dashboard
JSON against what the generators produce, and evaluates both modules.

Also fixes smart-selftest emitting a placeholder identity line when smartctl
cannot open a device. smartctl exits 0 with well-formed but empty JSON in that
case, so "SSD - unknown model - no serial" was reaching a dashboard row title;
it now emits nothing and lets the smartctl:ident fallback apply.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ

1 files changed, 109 insertions, 0 deletions

diff --git a/flake.nix b/flake.nix
new file mode 100644
index 0000000..1a9c661
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,109 @@
+{
+  description = "Host telemetry collectors and a Prometheus/Loki/Grafana server, as two separable NixOS modules";
+
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+
+  outputs = { self, nixpkgs }:
+    let
+      systems = [ "x86_64-linux" "aarch64-linux" ];
+      forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system});
+    in
+    {
+      # Paths rather than functions, so the modules are evaluated with the *consumer's*
+      # nixpkgs. This flake's own nixpkgs input therefore only affects packages, checks
+      # and devShells -- importing these modules never pulls in a second nixpkgs.
+      nixosModules = {
+        collectors = ./modules/collectors.nix;
+        server = ./modules/server.nix;
+        default = ./modules;
+      };
+
+      # The textfile collectors, runnable by hand. Both take --output PATH and write
+      # Prometheus exposition format; running them against a scratch file is how you
+      # check what a machine actually reports.
+      packages = forAllSystems (pkgs:
+        # callPackages, not callPackage: the file returns an attrset of
+        # derivations, and callPackage would wrap the whole set in
+        # override/overrideDerivation attrs that are not derivations.
+        let collectors = pkgs.callPackages ./modules/packages.nix { };
+        in collectors // { default = collectors.zfs-metrics; }
+      );
+
+      checks = forAllSystems (pkgs: {
+        # promtool over the recording rules. The server module also runs this at build
+        # time via services.prometheus.checkConfig, but that only fires for people who
+        # actually deploy it.
+        prometheus-rules = pkgs.runCommand "check-prometheus-rules"
+          { nativeBuildInputs = [ pkgs.prometheus.cli ]; }
+          ''
+            promtool check rules ${./rules}/*.yml
+            touch "$out"
+          '';
+
+        # The dashboard JSON is generated by the scripts next to it and committed, so it
+        # can rot silently if someone edits a generator without re-running it. Regenerate
+        # here and diff.
+        dashboards-current = pkgs.runCommand "check-dashboards-current"
+          { nativeBuildInputs = [ pkgs.python3 pkgs.diffutils ]; }
+          ''
+            cp -r ${./dashboards} work
+            chmod -R u+w work
+            cd work
+
+            # Note the loop variable is not called "out": that is the derivation's own
+            # output path, and shadowing it makes the final touch silently write
+            # somewhere else and the build fail with "failed to produce output path".
+            check() {
+              local generator="$1" committed="$2"
+              python3 "$generator" > regenerated.json
+              if ! diff -u "$committed" regenerated.json; then
+                echo
+                echo "dashboards/$committed is stale."
+                echo "Re-run: python3 dashboards/$generator > dashboards/$committed"
+                exit 1
+              fi
+            }
+
+            check gen_smart.py smart.json
+            check gen_zfs.py zfs-pools.json
+            touch "$out"
+          '';
+
+        # Both halves have to evaluate, separately and together. Catches option typos
+        # and missing mkIf guards without needing a real machine.
+        modules-eval = (nixpkgs.lib.nixosSystem {
+          system = pkgs.stdenv.hostPlatform.system;
+          modules = [
+            self.nixosModules.default
+            ({ lib, ... }: {
+              services.telemetry.collectors.enable = true;
+              services.telemetry.server = {
+                enable = true;
+                collectorHosts = [ "one" "two" ];
+                proxy.enable = true;
+                # Never read during evaluation; only has to satisfy the assertion.
+                grafana.secretKeyFile = "/run/secrets/grafana-secret-key";
+              };
+              boot.loader.grub.devices = [ "/dev/sda" ];
+              fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; };
+              system.stateVersion = "25.11";
+            })
+          ];
+        }).config.system.build.toplevel;
+      });
+
+      devShells = forAllSystems (pkgs: {
+        default = pkgs.mkShell {
+          packages = [
+            pkgs.python3 # the dashboard generators and collectors
+            pkgs.prometheus.cli # promtool, for the recording rules
+            pkgs.prometheus # a real Prometheus to run dashboard queries against
+            pkgs.jq
+            pkgs.smartmontools
+          ];
+        };
+      });
+
+      formatter = forAllSystems (pkgs: pkgs.nixpkgs-fmt);
+    };
+}