From d2d1386abb358fa547a40fc8df613d799155b56d Mon Sep 17 00:00:00 2001 From: Tim Keller Date: Thu, 27 Aug 2026 12:45:44 -0500 Subject: split into a flake with separable collectors/server modules Replaces the single lgtm.nix with a flake exposing two NixOS modules under services.telemetry, so the web services can move into a container declared in the consuming config rather than one hardcoded here. modules/collectors.nix host-side: node + smartctl exporters, the ZFS and SMART textfile collector timers, Alloy journal shipping. Must run on the host -- reads /proc, /sys, raw block devices and ZFS ioctls. modules/server.nix Prometheus, Loki, Grafana, optional Caddy. modules/common.nix ports, shared by both halves. modules/dashboards.nix jq fixup and grafana.com fetches. modules/packages.nix the two collectors as ordinary packages. Modules are exported as paths, not functions, so they evaluate with the consumer's nixpkgs; the flake's own input only affects packages/checks/devShells. Multi-host support. collectorHosts is a list, both scrape jobs relabel __address__ into a `host` label, and all 133 queries in the two generated dashboards are scoped to host=~"$host" behind a single-select Host picker. `instance` could not serve as that key: one machine is scraped twice on different ports (textfile metrics on :9100, smartctl_* on :9633) and the recording rules join across that boundary. Grafana's secret key becomes a plain secretKeyFile path wired through LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No sops-nix input. nix flake check runs promtool over the rules, diffs the committed dashboard JSON against what the generators produce, and evaluates both modules. Also fixes smart-selftest emitting a placeholder identity line when smartctl cannot open a device. smartctl exits 0 with well-formed but empty JSON in that case, so "SSD - unknown model - no serial" was reaching a dashboard row title; it now emits nothing and lets the smartctl:ident fallback apply. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ --- flake.nix | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 flake.nix (limited to 'flake.nix') diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..1a9c661 --- /dev/null +++ b/flake.nix @@ -0,0 +1,109 @@ +{ + description = "Host telemetry collectors and a Prometheus/Loki/Grafana server, as two separable NixOS modules"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + outputs = { self, nixpkgs }: + let + systems = [ "x86_64-linux" "aarch64-linux" ]; + forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system}); + in + { + # Paths rather than functions, so the modules are evaluated with the *consumer's* + # nixpkgs. This flake's own nixpkgs input therefore only affects packages, checks + # and devShells -- importing these modules never pulls in a second nixpkgs. + nixosModules = { + collectors = ./modules/collectors.nix; + server = ./modules/server.nix; + default = ./modules; + }; + + # The textfile collectors, runnable by hand. Both take --output PATH and write + # Prometheus exposition format; running them against a scratch file is how you + # check what a machine actually reports. + packages = forAllSystems (pkgs: + # callPackages, not callPackage: the file returns an attrset of + # derivations, and callPackage would wrap the whole set in + # override/overrideDerivation attrs that are not derivations. + let collectors = pkgs.callPackages ./modules/packages.nix { }; + in collectors // { default = collectors.zfs-metrics; } + ); + + checks = forAllSystems (pkgs: { + # promtool over the recording rules. The server module also runs this at build + # time via services.prometheus.checkConfig, but that only fires for people who + # actually deploy it. + prometheus-rules = pkgs.runCommand "check-prometheus-rules" + { nativeBuildInputs = [ pkgs.prometheus.cli ]; } + '' + promtool check rules ${./rules}/*.yml + touch "$out" + ''; + + # The dashboard JSON is generated by the scripts next to it and committed, so it + # can rot silently if someone edits a generator without re-running it. Regenerate + # here and diff. + dashboards-current = pkgs.runCommand "check-dashboards-current" + { nativeBuildInputs = [ pkgs.python3 pkgs.diffutils ]; } + '' + cp -r ${./dashboards} work + chmod -R u+w work + cd work + + # Note the loop variable is not called "out": that is the derivation's own + # output path, and shadowing it makes the final touch silently write + # somewhere else and the build fail with "failed to produce output path". + check() { + local generator="$1" committed="$2" + python3 "$generator" > regenerated.json + if ! diff -u "$committed" regenerated.json; then + echo + echo "dashboards/$committed is stale." + echo "Re-run: python3 dashboards/$generator > dashboards/$committed" + exit 1 + fi + } + + check gen_smart.py smart.json + check gen_zfs.py zfs-pools.json + touch "$out" + ''; + + # Both halves have to evaluate, separately and together. Catches option typos + # and missing mkIf guards without needing a real machine. + modules-eval = (nixpkgs.lib.nixosSystem { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + self.nixosModules.default + ({ lib, ... }: { + services.telemetry.collectors.enable = true; + services.telemetry.server = { + enable = true; + collectorHosts = [ "one" "two" ]; + proxy.enable = true; + # Never read during evaluation; only has to satisfy the assertion. + grafana.secretKeyFile = "/run/secrets/grafana-secret-key"; + }; + boot.loader.grub.devices = [ "/dev/sda" ]; + fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; }; + system.stateVersion = "25.11"; + }) + ]; + }).config.system.build.toplevel; + }); + + devShells = forAllSystems (pkgs: { + default = pkgs.mkShell { + packages = [ + pkgs.python3 # the dashboard generators and collectors + pkgs.prometheus.cli # promtool, for the recording rules + pkgs.prometheus # a real Prometheus to run dashboard queries against + pkgs.jq + pkgs.smartmontools + ]; + }; + }); + + formatter = forAllSystems (pkgs: pkgs.nixpkgs-fmt); + }; +} -- cgit v1.3.1