split into a flake with separable collectors/server modules

Replaces the single lgtm.nix with a flake exposing two NixOS modules under
services.telemetry, so the web services can move into a container declared in
the consuming config rather than one hardcoded here.

  modules/collectors.nix  host-side: node + smartctl exporters, the ZFS and
                          SMART textfile collector timers, Alloy journal
                          shipping. Must run on the host -- reads /proc, /sys,
                          raw block devices and ZFS ioctls.
  modules/server.nix      Prometheus, Loki, Grafana, optional Caddy.
  modules/common.nix      ports, shared by both halves.
  modules/dashboards.nix  jq fixup and grafana.com fetches.
  modules/packages.nix    the two collectors as ordinary packages.

Modules are exported as paths, not functions, so they evaluate with the
consumer's nixpkgs; the flake's own input only affects packages/checks/devShells.

Multi-host support. collectorHosts is a list, both scrape jobs relabel
__address__ into a `host` label, and all 133 queries in the two generated
dashboards are scoped to host=~"$host" behind a single-select Host picker.
`instance` could not serve as that key: one machine is scraped twice on
different ports (textfile metrics on :9100, smartctl_* on :9633) and the
recording rules join across that boundary.

Grafana's secret key becomes a plain secretKeyFile path wired through
LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No
sops-nix input.

nix flake check runs promtool over the rules, diffs the committed dashboard
JSON against what the generators produce, and evaluates both modules.

Also fixes smart-selftest emitting a placeholder identity line when smartctl
cannot open a device. smartctl exits 0 with well-formed but empty JSON in that
case, so "SSD - unknown model - no serial" was reaching a dashboard row title;
it now emits nothing and lets the smartctl:ident fallback apply.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ

1 files changed, 0 insertions, 534 deletions

diff --git a/lgtm.nix b/lgtm.nix
deleted file mode 100644
index a4144ec..0000000
--- a/lgtm.nix
+++ /dev/null
@@ -1,534 +0,0 @@
-{ pkgs, config, lib, ... }:
-
-let
-	# The hostname the container uses to reach the host's exporters.
-	hostName = "poweredge";
-
-	# One scrape interval to rule them all. This MUST match the Grafana Prometheus
-	# datasource's `timeInterval` below, otherwise Grafana computes $__rate_interval
-	# from the wrong assumption and every rate()-based panel intermittently renders
-	# "No data". (Prometheus' own default is 1m, which is what broke the Node Exporter
-	# Full dashboard: $__rate_interval resolved to 60s, so rate(x[60s]) over 60s-spaced
-	# samples usually saw fewer than the two samples rate() requires.)
-	scrapeInterval = "15s";
-
-	# Where the host's zpool collector drops its metrics for node_exporter to pick up.
-	textfileDir = "/var/lib/prometheus-node-exporter-textfile";
-
-	ports = {
-		node     = 9100;
-		smartctl = 9633;
-		loki     = 3100;
-		prometheus = 9090;
-		grafana  = 3000;
-	};
-
-	# Stable datasource UIDs, referenced by the dashboards below.
-	dsUid = { prometheus = "prometheus"; loki = "loki"; };
-
-	##########################################################################
-	# Dashboard plumbing
-	##########################################################################
-
-	# Dashboards exported from grafana.com carry an `__inputs` block and reference
-	# their datasource as "${DS_PROMETHEUS}". That placeholder is only substituted by
-	# Grafana's interactive *import* flow — file-based provisioning never runs it, so
-	# the uid stays literal and every panel fails to resolve a datasource. This filter
-	# performs the substitution at build time, using our fixed datasource UIDs.
-	dashboardFixup = pkgs.writeText "grafana-dashboard-fixup.jq" ''
-		def dsuid($pluginId):
-			if   $pluginId == "prometheus" then $prom
-			elif $pluginId == "loki"       then $loki
-			else "" end;
-
-		def subst($name; $uid):
-			walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end);
-
-		reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId)))
-		| del(.__inputs, .__requires)
-		# Provisioned dashboards must not carry an id, or Grafana rejects them on
-		# collision with whatever already occupies that id in its database.
-		| .id = null
-	'';
-
-	fixupDashboard = name: src: pkgs.runCommand "grafana-dashboard-${name}.json"
-		{ nativeBuildInputs = [ pkgs.jq ]; }
-		''
-			jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \
-			   --arg loki ${lib.escapeShellArg dsUid.loki} \
-			   -f ${dashboardFixup} < ${src} > "$out"
-		'';
-
-	# grafana.com dashboard, pinned by revision.
-	mkGrafanaComDashboard = { name, id, rev, sha256 }:
-		fixupDashboard name (pkgs.fetchurl {
-			url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download";
-			inherit sha256;
-		});
-
-	dashboards = {
-		# https://grafana.com/grafana/dashboards/1860 — Node Exporter Full
-		node-exporter-full = mkGrafanaComDashboard {
-			name = "node-exporter-full"; id = 1860; rev = 45;
-			sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY=";
-		};
-
-		# https://grafana.com/grafana/dashboards/22604 — Smartctl Exporter
-		smartctl-exporter = mkGrafanaComDashboard {
-			name = "smartctl-exporter"; id = 22604; rev = 2;
-			sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88=";
-		};
-
-		# Dashboard 15362 ("ZFS Pool Metrics") used to live here. It is an *InfluxDB*
-		# dashboard — its queries are InfluxQL against a `zpool_stats` measurement — so
-		# it can never render against Prometheus, no matter how it is provisioned. This
-		# is a Prometheus-native replacement built on the metrics we actually export:
-		# node_exporter's zfs collector plus the zpool textfile collector below.
-		zfs-pools = fixupDashboard "zfs-pools" ./dashboards/zfs-pools.json;
-
-		# CrystalDiskInfo-style per-drive SMART view. Generated by
-		# ./dashboards/gen_smart.py; the Good/Caution/Bad verdict it displays comes
-		# from the recording rules in ./rules/smartctl.yml.
-		smart = fixupDashboard "smart" ./dashboards/smart.json;
-	};
-
-	##########################################################################
-	# ZFS metrics (host)
-	##########################################################################
-
-	# node_exporter's zfs collector reads /proc/spl/kstat/zfs, which gives us ARC
-	# statistics, per-dataset objset counters and pool state -- but nothing about
-	# capacity, fragmentation, vdev errors, scrub history, compression, encryption or
-	# snapshots. OpenZFS 2.2+ also no longer publishes the per-pool `io` kstat, so
-	# node_zfs_zpool_* is largely empty. Everything else comes from the zpool/zfs CLIs
-	# via ./collectors/zfs-metrics.py, scraped through the textfile collector.
-	#
-	# The wrapper exists only to put the matching zfs userland on PATH; the collector
-	# itself is a plain script so it can be run and tested by hand.
-	snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ];
-
-	retentionArgs = lib.optionals config.services.zfs.autoSnapshot.enable (
-		lib.concatMap (label: [
-			"--retention"
-			"${label}=${toString config.services.zfs.autoSnapshot.${label}}"
-		]) snapshotLabels
-	);
-
-	zfsCollector = pkgs.writeShellApplication {
-		name = "zfs-textfile-collector";
-		runtimeInputs = [ config.boot.zfs.package ];
-		text = ''
-			exec ${pkgs.python3}/bin/python3 ${./collectors/zfs-metrics.py} \
-				--output ${textfileDir}/zfs.prom ${lib.escapeShellArgs retentionArgs}
-		'';
-	};
-
-	# smartctl_exporter reads only the entry *count* out of each drive's self-test log
-	# and discards the log table, so it cannot say when a drive last ran a short or
-	# extended test, or whether it passed. This parses `smartctl -j -l selftest`
-	# directly. Device labels are bare names ("sda") so they join against the
-	# exporter's own series.
-	smartSelftestCollector = pkgs.writeShellApplication {
-		name = "smart-selftest-collector";
-		runtimeInputs = [ pkgs.smartmontools ];
-		text = ''
-			exec ${pkgs.python3}/bin/python3 ${./collectors/smart-selftest.py} \
-				--output ${textfileDir}/smart-selftest.prom
-		'';
-	};
-in
-{
-	##########################################################################
-	# Host: log shipping
-	##########################################################################
-
-	services.alloy.enable = true;
-
-	environment.etc."alloy/config.alloy".text = ''
-		loki.source.journal "journal" {
-			forward_to    = [loki.write.default.receiver]
-			relabel_rules = loki.relabel.journal.rules
-			max_age       = "12h"
-		}
-
-		loki.relabel "journal" {
-			forward_to = []
-			rule {
-				source_labels = ["__journal__systemd_unit"]
-				target_label  = "unit"
-			}
-			rule {
-				source_labels = ["__journal__hostname"]
-				target_label  = "host"
-			}
-			rule {
-				source_labels = ["__journal_priority_keyword"]
-				target_label  = "level"
-			}
-		}
-
-		loki.write "default" {
-			endpoint {
-				url = "http://lgtm:${toString ports.loki}/loki/api/v1/push"
-			}
-		}
-	'';
-
-	##########################################################################
-	# Host: telemetry sources
-	#
-	# These have to live on the host, not in the container: they read host
-	# hardware and kernel state (/proc, /sys, SMART, ZFS ioctls) that a
-	# container has no view of. Only the web services (Grafana, Prometheus,
-	# Loki) run in the container.
-	##########################################################################
-
-	services.prometheus.exporters.node = {
-		enable = true;
-		port = ports.node;
-		listenAddress = "0.0.0.0";
-
-		# NOTE: enabledCollectors *adds* to node_exporter's defaults — cpu, diskstats,
-		# filesystem, meminfo, zfs, hwmon, pressure and friends are already on, so
-		# listing them here did nothing. Only genuinely opt-in collectors belong here.
-		enabledCollectors = [
-			"systemd"    # node_systemd_* — the "Systemd" row
-			"processes"  # node_processes_* — "Processes Detailed States", "PIDs", "Threads"
-			# "interrupts"  # fills the "IRQ Detail" panel, but emits one series per
-			#               # (cpu, irq) pair — thousands of series on a many-core box.
-			#               # Enable only if you want that one panel badly enough.
-		];
-
-		extraFlags = [
-			"--collector.textfile.directory=${textfileDir}"
-			# node_systemd_unit_start_time_seconds, so the dashboards can show when the
-			# zfs-snapshot-*/zfs-scrub units last ran rather than only whether they failed.
-			"--collector.systemd.enable-start-time-metrics"
-		];
-	};
-
-	# Upstream hardens the unit with ProtectHome=true, which makes /home look empty and
-	# blanks out the filesystem panels for any dataset mounted under it. read-only still
-	# allows statfs while keeping the sandbox.
-	systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only";
-
-	services.prometheus.exporters.smartctl = {
-		enable = true;
-		port = ports.smartctl;
-		listenAddress = "0.0.0.0";
-		# Disks are direct-attached (AHCI/HBA), so auto-discovery via `smartctl --scan`
-		# finds them; no explicit `devices` list needed. Throttle how often we actually
-		# shell out to smartctl — SMART data changes slowly and polling spins up disks.
-		maxInterval = "60s";
-	};
-
-	# SMART self-test history. Reading the log does not spin an idle disk up, but the
-	# data only changes when a test finishes, so this runs far less often than the
-	# exporter scrapes.
-	systemd.services.smart-selftest-collector = {
-		description = "Collect SMART self-test history for the Prometheus node exporter";
-		serviceConfig = {
-			Type = "oneshot";
-			ExecStart = lib.getExe smartSelftestCollector;
-			User = "root";
-			StateDirectory = "prometheus-node-exporter-textfile";
-			StateDirectoryMode = "0755";
-			UMask = "0022";
-			TimeoutStartSec = "180s";
-
-			# Same shape as the smartctl exporter's own hardening: it needs raw device
-			# access to talk to the disks, and nothing else.
-			AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
-			CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
-			DevicePolicy = "closed";
-			DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ];
-			LockPersonality = true;
-			MemoryDenyWriteExecute = true;
-			NoNewPrivileges = true;
-			PrivateNetwork = true;
-			PrivateTmp = true;
-			ProtectControlGroups = true;
-			ProtectHome = true;
-			ProtectHostname = true;
-			ProtectKernelLogs = true;
-			ProtectKernelModules = true;
-			ProtectSystem = "strict";
-			RestrictAddressFamilies = [ "AF_UNIX" ];
-			RestrictNamespaces = true;
-			RestrictRealtime = true;
-			RestrictSUIDSGID = true;
-			SystemCallArchitectures = "native";
-		};
-	};
-
-	systemd.timers.smart-selftest-collector = {
-		description = "Collect SMART self-test history periodically";
-		wantedBy = [ "timers.target" ];
-		timerConfig = {
-			OnBootSec = "3min";
-			OnUnitActiveSec = "15min";
-			AccuracySec = "1min";
-			Unit = "smart-selftest-collector.service";
-		};
-	};
-
-	# Pool capacity and health, vdev errors, scrub history, dataset properties and
-	# snapshot retention. See zfsCollector above for why node_exporter's built-in zfs
-	# collector isn't enough on its own.
-	systemd.services.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled {
-		description = "Collect ZFS metrics for the Prometheus node exporter";
-		serviceConfig = {
-			Type = "oneshot";
-			ExecStart = lib.getExe zfsCollector;
-			User = "root";
-			StateDirectory = "prometheus-node-exporter-textfile";
-			StateDirectoryMode = "0755";
-			UMask = "0022";  # node_exporter runs as a different user and must read these
-			# Listing every snapshot on a busy pool is the slow part; keep this well
-			# above the subprocess timeout inside the collector.
-			TimeoutStartSec = "180s";
-
-			# Hardening — this only needs to read ZFS state and write one file.
-			# CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated
-			# on it; everything else is dropped.
-			CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ];
-			LockPersonality = true;
-			MemoryDenyWriteExecute = true;
-			NoNewPrivileges = true;
-			PrivateNetwork = true;
-			PrivateTmp = true;
-			ProtectControlGroups = true;
-			ProtectHome = true;
-			ProtectHostname = true;
-			ProtectKernelLogs = true;
-			ProtectKernelModules = true;
-			ProtectSystem = "strict";
-			RestrictAddressFamilies = [ "AF_UNIX" ];
-			RestrictNamespaces = true;
-			RestrictRealtime = true;
-			RestrictSUIDSGID = true;
-			SystemCallArchitectures = "native";
-			SystemCallFilter = [ "@system-service" ];
-		};
-	};
-
-	systemd.timers.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled {
-		description = "Collect ZFS metrics periodically";
-		wantedBy = [ "timers.target" ];
-		timerConfig = {
-			OnBootSec = "1min";
-			OnUnitActiveSec = "1min";
-			AccuracySec = "5s";
-			Unit = "zfs-textfile-collector.service";
-		};
-	};
-
-	# The collector used to write zpool.prom. Nothing removes a stale textfile once its
-	# writer is gone, and node_exporter would happily keep serving those frozen metrics
-	# forever, so delete it on activation.
-	systemd.tmpfiles.rules = lib.mkIf config.boot.zfs.enabled [
-		"r ${textfileDir}/zpool.prom"
-	];
-
-	# Reachable from the container, which sits on br-lan0 with its own address.
-	# (Port 9273 is gone — that was telegraf/zpool_influxdb, replaced by the textfile
-	# collector above, which needs no listener of its own.)
-	networking.firewall.allowedTCPPorts = [ ports.node ports.smartctl ];
-
-	##########################################################################
-	# Secrets
-	##########################################################################
-
-	sops.secrets.grafana-secret-key = {
-		sopsFile = ./resources/secrets/grafana.yaml;
-		key = "secret_key";
-	};
-
-	##########################################################################
-	# Container: Grafana / Prometheus / Loki
-	##########################################################################
-
-	containers.grafana = {
-		autoStart = true;
-		privateNetwork = true;
-		hostBridge = "br-lan0";
-		localMacAddress = "02:00:00:00:00:06";
-
-		bindMounts."/run/secrets/grafana-secret_key" = {
-			hostPath = config.sops.secrets.grafana-secret-key.path;
-			isReadOnly = true;
-		};
-
-		config = { lib, pkgs, config, ... }: {
-			# Network
-			networking.enableIPv6 = false;
-			networking.interfaces.eth0.useDHCP = true;
-			networking.firewall.allowedTCPPorts = [
-				80                 # Caddy -> Grafana
-				ports.loki         # host's Alloy pushes here
-				ports.prometheus   # handy for debugging scrape health directly
-			];
-
-			# ---------------------------------------------------------------
-			# Loki
-			# ---------------------------------------------------------------
-			services.loki = {
-				enable = true;
-				configuration = {
-					server = {
-						http_listen_port = ports.loki;
-						# Explicit, since the host's Alloy needs to reach it to push logs.
-						http_listen_address = "0.0.0.0";
-					};
-					auth_enabled = false;
-
-					common = {
-						path_prefix = "/var/lib/loki";
-						storage.filesystem = {
-							chunks_directory = "/var/lib/loki/chunks";
-							rules_directory = "/var/lib/loki/rules";
-						};
-						ring = {
-							instance_addr = "127.0.0.1";
-							kvstore.store = "inmemory";
-						};
-						replication_factor = 1;
-					};
-
-					schema_config.configs = [{
-						from = "2024-01-01";
-						store = "tsdb";
-						object_store = "filesystem";
-						schema = "v13";
-						index = { prefix = "index_"; period = "24h"; };
-					}];
-				};
-			};
-
-			# ---------------------------------------------------------------
-			# Prometheus
-			# ---------------------------------------------------------------
-			services.prometheus = {
-				enable = true;
-
-				# Without this Prometheus falls back to its built-in 1m scrape interval,
-				# which desynchronises from what Grafana assumes and breaks every
-				# rate()-based panel. See the comment on `scrapeInterval` at the top.
-				globalConfig = {
-					scrape_interval = scrapeInterval;
-					evaluation_interval = scrapeInterval;
-					scrape_timeout = "10s";
-				};
-
-				# Drive health verdict. Kept as recording rules rather than inlined in
-				# dashboard panels so the logic is written once and reviewable on its
-				# own. services.prometheus.checkConfig runs promtool at build time, so
-				# a broken rule fails the build instead of the service.
-				ruleFiles = [ ./rules/smartctl.yml ];
-
-				scrapeConfigs = [
-					{
-						job_name = "node";
-						static_configs = [{ targets = [ "${hostName}:${toString ports.node}" ]; }];
-					}
-					{
-						job_name = "smartctl";
-						# Querying SMART is comparatively slow and the data barely moves, so
-						# this job opts out of the global 15s. Safe to do here because the
-						# smartctl dashboard uses no rate() queries at all.
-						scrape_interval = "60s";
-						scrape_timeout = "30s";
-						static_configs = [{ targets = [ "${hostName}:${toString ports.smartctl}" ]; }];
-					}
-				];
-			};
-
-			# ---------------------------------------------------------------
-			# Grafana
-			# https://wiki.nixos.org/wiki/Grafana
-			# ---------------------------------------------------------------
-			services.grafana = {
-				enable = true;
-				settings.server = { http_addr = "0.0.0.0"; http_port = ports.grafana; };
-
-				provision = {
-					enable = true;
-
-					# Mutable dashboard provider pulling from /etc/grafana-dashboards, so
-					# dashboards can be declared with environment.etc below.
-					dashboards.settings.providers = [{
-						name = "my dashboards";
-						disableDeletion = true;
-						options = {
-							path = "/etc/grafana-dashboards";
-							foldersFromFilesStructure = true;
-						};
-					}];
-
-					# Grafana cannot change the UID of an existing datasource in place. Its
-					# provisioner finds the datasource by name, then calls UpdateDataSource,
-					# which re-fetches using `id AND uid` — with a new uid that row doesn't
-					# match, so it fails with "data source not found" and Grafana refuses to
-					# start. Deleting first turns the upsert into a plain insert; the
-					# provisioner runs deletes before inserts for exactly this case, and
-					# suppresses the deletion event because the datasource is about to be
-					# re-created. Safe to leave in permanently: both datasources are fully
-					# declarative, and dashboards bind to them by UID, not by database ID.
-					datasources.settings.deleteDatasources = [
-						{ name = "Prometheus"; orgId = 1; }
-						{ name = "Loki"; orgId = 1; }
-					];
-
-					datasources.settings.datasources = [
-						{
-							name = "Prometheus";
-							type = "prometheus";
-							uid = dsUid.prometheus;  # fixed, so dashboards can reference it
-							url = "http://127.0.0.1:${toString ports.prometheus}";
-							isDefault = true;
-							jsonData = {
-								# Tells Grafana how often the data is actually scraped. This is
-								# what $__rate_interval is derived from; leaving it unset makes
-								# Grafana guess 15s and mis-size every rate() window.
-								timeInterval = scrapeInterval;
-								httpMethod = "POST";
-							};
-						}
-						{
-							name = "Loki";
-							type = "loki";
-							uid = dsUid.loki;
-							url = "http://127.0.0.1:${toString ports.loki}";
-						}
-					];
-				};
-			};
-
-			# Grafana secret key
-			systemd.services.grafana.serviceConfig.LoadCredential = [
-				"secret_key:/run/secrets/grafana-secret_key"
-			];
-			services.grafana.settings.security.secret_key =
-				"$__file{/run/credentials/grafana.service/secret_key}";
-
-			# ---------------------------------------------------------------
-			# Dashboards
-			# ---------------------------------------------------------------
-			environment.etc = lib.mapAttrs' (name: drv:
-				lib.nameValuePair "grafana-dashboards/${name}.json" { source = drv; }
-			) dashboards;
-
-			# ---------------------------------------------------------------
-			# Reverse proxy
-			# ---------------------------------------------------------------
-			services.caddy = {
-				enable = true;
-				virtualHosts.":80".extraConfig = ''
-					reverse_proxy localhost:${toString ports.grafana}
-				'';
-			};
-
-			system.stateVersion = "26.05";
-		};
-	};
-}