initial commit

1 files changed, 534 insertions, 0 deletions

diff --git a/lgtm.nix b/lgtm.nix
new file mode 100644
index 0000000..a4144ec
--- /dev/null
+++ b/lgtm.nix
@@ -0,0 +1,534 @@
+{ pkgs, config, lib, ... }:
+
+let
+	# The hostname the container uses to reach the host's exporters.
+	hostName = "poweredge";
+
+	# One scrape interval to rule them all. This MUST match the Grafana Prometheus
+	# datasource's `timeInterval` below, otherwise Grafana computes $__rate_interval
+	# from the wrong assumption and every rate()-based panel intermittently renders
+	# "No data". (Prometheus' own default is 1m, which is what broke the Node Exporter
+	# Full dashboard: $__rate_interval resolved to 60s, so rate(x[60s]) over 60s-spaced
+	# samples usually saw fewer than the two samples rate() requires.)
+	scrapeInterval = "15s";
+
+	# Where the host's zpool collector drops its metrics for node_exporter to pick up.
+	textfileDir = "/var/lib/prometheus-node-exporter-textfile";
+
+	ports = {
+		node     = 9100;
+		smartctl = 9633;
+		loki     = 3100;
+		prometheus = 9090;
+		grafana  = 3000;
+	};
+
+	# Stable datasource UIDs, referenced by the dashboards below.
+	dsUid = { prometheus = "prometheus"; loki = "loki"; };
+
+	##########################################################################
+	# Dashboard plumbing
+	##########################################################################
+
+	# Dashboards exported from grafana.com carry an `__inputs` block and reference
+	# their datasource as "${DS_PROMETHEUS}". That placeholder is only substituted by
+	# Grafana's interactive *import* flow — file-based provisioning never runs it, so
+	# the uid stays literal and every panel fails to resolve a datasource. This filter
+	# performs the substitution at build time, using our fixed datasource UIDs.
+	dashboardFixup = pkgs.writeText "grafana-dashboard-fixup.jq" ''
+		def dsuid($pluginId):
+			if   $pluginId == "prometheus" then $prom
+			elif $pluginId == "loki"       then $loki
+			else "" end;
+
+		def subst($name; $uid):
+			walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end);
+
+		reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId)))
+		| del(.__inputs, .__requires)
+		# Provisioned dashboards must not carry an id, or Grafana rejects them on
+		# collision with whatever already occupies that id in its database.
+		| .id = null
+	'';
+
+	fixupDashboard = name: src: pkgs.runCommand "grafana-dashboard-${name}.json"
+		{ nativeBuildInputs = [ pkgs.jq ]; }
+		''
+			jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \
+			   --arg loki ${lib.escapeShellArg dsUid.loki} \
+			   -f ${dashboardFixup} < ${src} > "$out"
+		'';
+
+	# grafana.com dashboard, pinned by revision.
+	mkGrafanaComDashboard = { name, id, rev, sha256 }:
+		fixupDashboard name (pkgs.fetchurl {
+			url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download";
+			inherit sha256;
+		});
+
+	dashboards = {
+		# https://grafana.com/grafana/dashboards/1860 — Node Exporter Full
+		node-exporter-full = mkGrafanaComDashboard {
+			name = "node-exporter-full"; id = 1860; rev = 45;
+			sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY=";
+		};
+
+		# https://grafana.com/grafana/dashboards/22604 — Smartctl Exporter
+		smartctl-exporter = mkGrafanaComDashboard {
+			name = "smartctl-exporter"; id = 22604; rev = 2;
+			sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88=";
+		};
+
+		# Dashboard 15362 ("ZFS Pool Metrics") used to live here. It is an *InfluxDB*
+		# dashboard — its queries are InfluxQL against a `zpool_stats` measurement — so
+		# it can never render against Prometheus, no matter how it is provisioned. This
+		# is a Prometheus-native replacement built on the metrics we actually export:
+		# node_exporter's zfs collector plus the zpool textfile collector below.
+		zfs-pools = fixupDashboard "zfs-pools" ./dashboards/zfs-pools.json;
+
+		# CrystalDiskInfo-style per-drive SMART view. Generated by
+		# ./dashboards/gen_smart.py; the Good/Caution/Bad verdict it displays comes
+		# from the recording rules in ./rules/smartctl.yml.
+		smart = fixupDashboard "smart" ./dashboards/smart.json;
+	};
+
+	##########################################################################
+	# ZFS metrics (host)
+	##########################################################################
+
+	# node_exporter's zfs collector reads /proc/spl/kstat/zfs, which gives us ARC
+	# statistics, per-dataset objset counters and pool state -- but nothing about
+	# capacity, fragmentation, vdev errors, scrub history, compression, encryption or
+	# snapshots. OpenZFS 2.2+ also no longer publishes the per-pool `io` kstat, so
+	# node_zfs_zpool_* is largely empty. Everything else comes from the zpool/zfs CLIs
+	# via ./collectors/zfs-metrics.py, scraped through the textfile collector.
+	#
+	# The wrapper exists only to put the matching zfs userland on PATH; the collector
+	# itself is a plain script so it can be run and tested by hand.
+	snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ];
+
+	retentionArgs = lib.optionals config.services.zfs.autoSnapshot.enable (
+		lib.concatMap (label: [
+			"--retention"
+			"${label}=${toString config.services.zfs.autoSnapshot.${label}}"
+		]) snapshotLabels
+	);
+
+	zfsCollector = pkgs.writeShellApplication {
+		name = "zfs-textfile-collector";
+		runtimeInputs = [ config.boot.zfs.package ];
+		text = ''
+			exec ${pkgs.python3}/bin/python3 ${./collectors/zfs-metrics.py} \
+				--output ${textfileDir}/zfs.prom ${lib.escapeShellArgs retentionArgs}
+		'';
+	};
+
+	# smartctl_exporter reads only the entry *count* out of each drive's self-test log
+	# and discards the log table, so it cannot say when a drive last ran a short or
+	# extended test, or whether it passed. This parses `smartctl -j -l selftest`
+	# directly. Device labels are bare names ("sda") so they join against the
+	# exporter's own series.
+	smartSelftestCollector = pkgs.writeShellApplication {
+		name = "smart-selftest-collector";
+		runtimeInputs = [ pkgs.smartmontools ];
+		text = ''
+			exec ${pkgs.python3}/bin/python3 ${./collectors/smart-selftest.py} \
+				--output ${textfileDir}/smart-selftest.prom
+		'';
+	};
+in
+{
+	##########################################################################
+	# Host: log shipping
+	##########################################################################
+
+	services.alloy.enable = true;
+
+	environment.etc."alloy/config.alloy".text = ''
+		loki.source.journal "journal" {
+			forward_to    = [loki.write.default.receiver]
+			relabel_rules = loki.relabel.journal.rules
+			max_age       = "12h"
+		}
+
+		loki.relabel "journal" {
+			forward_to = []
+			rule {
+				source_labels = ["__journal__systemd_unit"]
+				target_label  = "unit"
+			}
+			rule {
+				source_labels = ["__journal__hostname"]
+				target_label  = "host"
+			}
+			rule {
+				source_labels = ["__journal_priority_keyword"]
+				target_label  = "level"
+			}
+		}
+
+		loki.write "default" {
+			endpoint {
+				url = "http://lgtm:${toString ports.loki}/loki/api/v1/push"
+			}
+		}
+	'';
+
+	##########################################################################
+	# Host: telemetry sources
+	#
+	# These have to live on the host, not in the container: they read host
+	# hardware and kernel state (/proc, /sys, SMART, ZFS ioctls) that a
+	# container has no view of. Only the web services (Grafana, Prometheus,
+	# Loki) run in the container.
+	##########################################################################
+
+	services.prometheus.exporters.node = {
+		enable = true;
+		port = ports.node;
+		listenAddress = "0.0.0.0";
+
+		# NOTE: enabledCollectors *adds* to node_exporter's defaults — cpu, diskstats,
+		# filesystem, meminfo, zfs, hwmon, pressure and friends are already on, so
+		# listing them here did nothing. Only genuinely opt-in collectors belong here.
+		enabledCollectors = [
+			"systemd"    # node_systemd_* — the "Systemd" row
+			"processes"  # node_processes_* — "Processes Detailed States", "PIDs", "Threads"
+			# "interrupts"  # fills the "IRQ Detail" panel, but emits one series per
+			#               # (cpu, irq) pair — thousands of series on a many-core box.
+			#               # Enable only if you want that one panel badly enough.
+		];
+
+		extraFlags = [
+			"--collector.textfile.directory=${textfileDir}"
+			# node_systemd_unit_start_time_seconds, so the dashboards can show when the
+			# zfs-snapshot-*/zfs-scrub units last ran rather than only whether they failed.
+			"--collector.systemd.enable-start-time-metrics"
+		];
+	};
+
+	# Upstream hardens the unit with ProtectHome=true, which makes /home look empty and
+	# blanks out the filesystem panels for any dataset mounted under it. read-only still
+	# allows statfs while keeping the sandbox.
+	systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only";
+
+	services.prometheus.exporters.smartctl = {
+		enable = true;
+		port = ports.smartctl;
+		listenAddress = "0.0.0.0";
+		# Disks are direct-attached (AHCI/HBA), so auto-discovery via `smartctl --scan`
+		# finds them; no explicit `devices` list needed. Throttle how often we actually
+		# shell out to smartctl — SMART data changes slowly and polling spins up disks.
+		maxInterval = "60s";
+	};
+
+	# SMART self-test history. Reading the log does not spin an idle disk up, but the
+	# data only changes when a test finishes, so this runs far less often than the
+	# exporter scrapes.
+	systemd.services.smart-selftest-collector = {
+		description = "Collect SMART self-test history for the Prometheus node exporter";
+		serviceConfig = {
+			Type = "oneshot";
+			ExecStart = lib.getExe smartSelftestCollector;
+			User = "root";
+			StateDirectory = "prometheus-node-exporter-textfile";
+			StateDirectoryMode = "0755";
+			UMask = "0022";
+			TimeoutStartSec = "180s";
+
+			# Same shape as the smartctl exporter's own hardening: it needs raw device
+			# access to talk to the disks, and nothing else.
+			AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
+			CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
+			DevicePolicy = "closed";
+			DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ];
+			LockPersonality = true;
+			MemoryDenyWriteExecute = true;
+			NoNewPrivileges = true;
+			PrivateNetwork = true;
+			PrivateTmp = true;
+			ProtectControlGroups = true;
+			ProtectHome = true;
+			ProtectHostname = true;
+			ProtectKernelLogs = true;
+			ProtectKernelModules = true;
+			ProtectSystem = "strict";
+			RestrictAddressFamilies = [ "AF_UNIX" ];
+			RestrictNamespaces = true;
+			RestrictRealtime = true;
+			RestrictSUIDSGID = true;
+			SystemCallArchitectures = "native";
+		};
+	};
+
+	systemd.timers.smart-selftest-collector = {
+		description = "Collect SMART self-test history periodically";
+		wantedBy = [ "timers.target" ];
+		timerConfig = {
+			OnBootSec = "3min";
+			OnUnitActiveSec = "15min";
+			AccuracySec = "1min";
+			Unit = "smart-selftest-collector.service";
+		};
+	};
+
+	# Pool capacity and health, vdev errors, scrub history, dataset properties and
+	# snapshot retention. See zfsCollector above for why node_exporter's built-in zfs
+	# collector isn't enough on its own.
+	systemd.services.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled {
+		description = "Collect ZFS metrics for the Prometheus node exporter";
+		serviceConfig = {
+			Type = "oneshot";
+			ExecStart = lib.getExe zfsCollector;
+			User = "root";
+			StateDirectory = "prometheus-node-exporter-textfile";
+			StateDirectoryMode = "0755";
+			UMask = "0022";  # node_exporter runs as a different user and must read these
+			# Listing every snapshot on a busy pool is the slow part; keep this well
+			# above the subprocess timeout inside the collector.
+			TimeoutStartSec = "180s";
+
+			# Hardening — this only needs to read ZFS state and write one file.
+			# CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated
+			# on it; everything else is dropped.
+			CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ];
+			LockPersonality = true;
+			MemoryDenyWriteExecute = true;
+			NoNewPrivileges = true;
+			PrivateNetwork = true;
+			PrivateTmp = true;
+			ProtectControlGroups = true;
+			ProtectHome = true;
+			ProtectHostname = true;
+			ProtectKernelLogs = true;
+			ProtectKernelModules = true;
+			ProtectSystem = "strict";
+			RestrictAddressFamilies = [ "AF_UNIX" ];
+			RestrictNamespaces = true;
+			RestrictRealtime = true;
+			RestrictSUIDSGID = true;
+			SystemCallArchitectures = "native";
+			SystemCallFilter = [ "@system-service" ];
+		};
+	};
+
+	systemd.timers.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled {
+		description = "Collect ZFS metrics periodically";
+		wantedBy = [ "timers.target" ];
+		timerConfig = {
+			OnBootSec = "1min";
+			OnUnitActiveSec = "1min";
+			AccuracySec = "5s";
+			Unit = "zfs-textfile-collector.service";
+		};
+	};
+
+	# The collector used to write zpool.prom. Nothing removes a stale textfile once its
+	# writer is gone, and node_exporter would happily keep serving those frozen metrics
+	# forever, so delete it on activation.
+	systemd.tmpfiles.rules = lib.mkIf config.boot.zfs.enabled [
+		"r ${textfileDir}/zpool.prom"
+	];
+
+	# Reachable from the container, which sits on br-lan0 with its own address.
+	# (Port 9273 is gone — that was telegraf/zpool_influxdb, replaced by the textfile
+	# collector above, which needs no listener of its own.)
+	networking.firewall.allowedTCPPorts = [ ports.node ports.smartctl ];
+
+	##########################################################################
+	# Secrets
+	##########################################################################
+
+	sops.secrets.grafana-secret-key = {
+		sopsFile = ./resources/secrets/grafana.yaml;
+		key = "secret_key";
+	};
+
+	##########################################################################
+	# Container: Grafana / Prometheus / Loki
+	##########################################################################
+
+	containers.grafana = {
+		autoStart = true;
+		privateNetwork = true;
+		hostBridge = "br-lan0";
+		localMacAddress = "02:00:00:00:00:06";
+
+		bindMounts."/run/secrets/grafana-secret_key" = {
+			hostPath = config.sops.secrets.grafana-secret-key.path;
+			isReadOnly = true;
+		};
+
+		config = { lib, pkgs, config, ... }: {
+			# Network
+			networking.enableIPv6 = false;
+			networking.interfaces.eth0.useDHCP = true;
+			networking.firewall.allowedTCPPorts = [
+				80                 # Caddy -> Grafana
+				ports.loki         # host's Alloy pushes here
+				ports.prometheus   # handy for debugging scrape health directly
+			];
+
+			# ---------------------------------------------------------------
+			# Loki
+			# ---------------------------------------------------------------
+			services.loki = {
+				enable = true;
+				configuration = {
+					server = {
+						http_listen_port = ports.loki;
+						# Explicit, since the host's Alloy needs to reach it to push logs.
+						http_listen_address = "0.0.0.0";
+					};
+					auth_enabled = false;
+
+					common = {
+						path_prefix = "/var/lib/loki";
+						storage.filesystem = {
+							chunks_directory = "/var/lib/loki/chunks";
+							rules_directory = "/var/lib/loki/rules";
+						};
+						ring = {
+							instance_addr = "127.0.0.1";
+							kvstore.store = "inmemory";
+						};
+						replication_factor = 1;
+					};
+
+					schema_config.configs = [{
+						from = "2024-01-01";
+						store = "tsdb";
+						object_store = "filesystem";
+						schema = "v13";
+						index = { prefix = "index_"; period = "24h"; };
+					}];
+				};
+			};
+
+			# ---------------------------------------------------------------
+			# Prometheus
+			# ---------------------------------------------------------------
+			services.prometheus = {
+				enable = true;
+
+				# Without this Prometheus falls back to its built-in 1m scrape interval,
+				# which desynchronises from what Grafana assumes and breaks every
+				# rate()-based panel. See the comment on `scrapeInterval` at the top.
+				globalConfig = {
+					scrape_interval = scrapeInterval;
+					evaluation_interval = scrapeInterval;
+					scrape_timeout = "10s";
+				};
+
+				# Drive health verdict. Kept as recording rules rather than inlined in
+				# dashboard panels so the logic is written once and reviewable on its
+				# own. services.prometheus.checkConfig runs promtool at build time, so
+				# a broken rule fails the build instead of the service.
+				ruleFiles = [ ./rules/smartctl.yml ];
+
+				scrapeConfigs = [
+					{
+						job_name = "node";
+						static_configs = [{ targets = [ "${hostName}:${toString ports.node}" ]; }];
+					}
+					{
+						job_name = "smartctl";
+						# Querying SMART is comparatively slow and the data barely moves, so
+						# this job opts out of the global 15s. Safe to do here because the
+						# smartctl dashboard uses no rate() queries at all.
+						scrape_interval = "60s";
+						scrape_timeout = "30s";
+						static_configs = [{ targets = [ "${hostName}:${toString ports.smartctl}" ]; }];
+					}
+				];
+			};
+
+			# ---------------------------------------------------------------
+			# Grafana
+			# https://wiki.nixos.org/wiki/Grafana
+			# ---------------------------------------------------------------
+			services.grafana = {
+				enable = true;
+				settings.server = { http_addr = "0.0.0.0"; http_port = ports.grafana; };
+
+				provision = {
+					enable = true;
+
+					# Mutable dashboard provider pulling from /etc/grafana-dashboards, so
+					# dashboards can be declared with environment.etc below.
+					dashboards.settings.providers = [{
+						name = "my dashboards";
+						disableDeletion = true;
+						options = {
+							path = "/etc/grafana-dashboards";
+							foldersFromFilesStructure = true;
+						};
+					}];
+
+					# Grafana cannot change the UID of an existing datasource in place. Its
+					# provisioner finds the datasource by name, then calls UpdateDataSource,
+					# which re-fetches using `id AND uid` — with a new uid that row doesn't
+					# match, so it fails with "data source not found" and Grafana refuses to
+					# start. Deleting first turns the upsert into a plain insert; the
+					# provisioner runs deletes before inserts for exactly this case, and
+					# suppresses the deletion event because the datasource is about to be
+					# re-created. Safe to leave in permanently: both datasources are fully
+					# declarative, and dashboards bind to them by UID, not by database ID.
+					datasources.settings.deleteDatasources = [
+						{ name = "Prometheus"; orgId = 1; }
+						{ name = "Loki"; orgId = 1; }
+					];
+
+					datasources.settings.datasources = [
+						{
+							name = "Prometheus";
+							type = "prometheus";
+							uid = dsUid.prometheus;  # fixed, so dashboards can reference it
+							url = "http://127.0.0.1:${toString ports.prometheus}";
+							isDefault = true;
+							jsonData = {
+								# Tells Grafana how often the data is actually scraped. This is
+								# what $__rate_interval is derived from; leaving it unset makes
+								# Grafana guess 15s and mis-size every rate() window.
+								timeInterval = scrapeInterval;
+								httpMethod = "POST";
+							};
+						}
+						{
+							name = "Loki";
+							type = "loki";
+							uid = dsUid.loki;
+							url = "http://127.0.0.1:${toString ports.loki}";
+						}
+					];
+				};
+			};
+
+			# Grafana secret key
+			systemd.services.grafana.serviceConfig.LoadCredential = [
+				"secret_key:/run/secrets/grafana-secret_key"
+			];
+			services.grafana.settings.security.secret_key =
+				"$__file{/run/credentials/grafana.service/secret_key}";
+
+			# ---------------------------------------------------------------
+			# Dashboards
+			# ---------------------------------------------------------------
+			environment.etc = lib.mapAttrs' (name: drv:
+				lib.nameValuePair "grafana-dashboards/${name}.json" { source = drv; }
+			) dashboards;
+
+			# ---------------------------------------------------------------
+			# Reverse proxy
+			# ---------------------------------------------------------------
+			services.caddy = {
+				enable = true;
+				virtualHosts.":80".extraConfig = ''
+					reverse_proxy localhost:${toString ports.grafana}
+				'';
+			};
+
+			system.stateVersion = "26.05";
+		};
+	};
+}