path: root/lgtm.nix

{ pkgs, config, lib, ... }:

let
	# The hostname the container uses to reach the host's exporters.
	hostName = "poweredge";

	# One scrape interval to rule them all. This MUST match the Grafana Prometheus
	# datasource's `timeInterval` below, otherwise Grafana computes $__rate_interval
	# from the wrong assumption and every rate()-based panel intermittently renders
	# "No data". (Prometheus' own default is 1m, which is what broke the Node Exporter
	# Full dashboard: $__rate_interval resolved to 60s, so rate(x[60s]) over 60s-spaced
	# samples usually saw fewer than the two samples rate() requires.)
	scrapeInterval = "15s";

	# Where the host's zpool collector drops its metrics for node_exporter to pick up.
	textfileDir = "/var/lib/prometheus-node-exporter-textfile";

	ports = {
		node     = 9100;
		smartctl = 9633;
		loki     = 3100;
		prometheus = 9090;
		grafana  = 3000;
	};

	# Stable datasource UIDs, referenced by the dashboards below.
	dsUid = { prometheus = "prometheus"; loki = "loki"; };

	##########################################################################
	# Dashboard plumbing
	##########################################################################

	# Dashboards exported from grafana.com carry an `__inputs` block and reference
	# their datasource as "${DS_PROMETHEUS}". That placeholder is only substituted by
	# Grafana's interactive *import* flow — file-based provisioning never runs it, so
	# the uid stays literal and every panel fails to resolve a datasource. This filter
	# performs the substitution at build time, using our fixed datasource UIDs.
	dashboardFixup = pkgs.writeText "grafana-dashboard-fixup.jq" ''
		def dsuid($pluginId):
			if   $pluginId == "prometheus" then $prom
			elif $pluginId == "loki"       then $loki
			else "" end;

		def subst($name; $uid):
			walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end);

		reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId)))
		| del(.__inputs, .__requires)
		# Provisioned dashboards must not carry an id, or Grafana rejects them on
		# collision with whatever already occupies that id in its database.
		| .id = null
	'';

	fixupDashboard = name: src: pkgs.runCommand "grafana-dashboard-${name}.json"
		{ nativeBuildInputs = [ pkgs.jq ]; }
		''
			jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \
			   --arg loki ${lib.escapeShellArg dsUid.loki} \
			   -f ${dashboardFixup} < ${src} > "$out"
		'';

	# grafana.com dashboard, pinned by revision.
	mkGrafanaComDashboard = { name, id, rev, sha256 }:
		fixupDashboard name (pkgs.fetchurl {
			url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download";
			inherit sha256;
		});

	dashboards = {
		# https://grafana.com/grafana/dashboards/1860 — Node Exporter Full
		node-exporter-full = mkGrafanaComDashboard {
			name = "node-exporter-full"; id = 1860; rev = 45;
			sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY=";
		};

		# https://grafana.com/grafana/dashboards/22604 — Smartctl Exporter
		smartctl-exporter = mkGrafanaComDashboard {
			name = "smartctl-exporter"; id = 22604; rev = 2;
			sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88=";
		};

		# Dashboard 15362 ("ZFS Pool Metrics") used to live here. It is an *InfluxDB*
		# dashboard — its queries are InfluxQL against a `zpool_stats` measurement — so
		# it can never render against Prometheus, no matter how it is provisioned. This
		# is a Prometheus-native replacement built on the metrics we actually export:
		# node_exporter's zfs collector plus the zpool textfile collector below.
		zfs-pools = fixupDashboard "zfs-pools" ./dashboards/zfs-pools.json;

		# CrystalDiskInfo-style per-drive SMART view. Generated by
		# ./dashboards/gen_smart.py; the Good/Caution/Bad verdict it displays comes
		# from the recording rules in ./rules/smartctl.yml.
		smart = fixupDashboard "smart" ./dashboards/smart.json;
	};

	##########################################################################
	# ZFS metrics (host)
	##########################################################################

	# node_exporter's zfs collector reads /proc/spl/kstat/zfs, which gives us ARC
	# statistics, per-dataset objset counters and pool state -- but nothing about
	# capacity, fragmentation, vdev errors, scrub history, compression, encryption or
	# snapshots. OpenZFS 2.2+ also no longer publishes the per-pool `io` kstat, so
	# node_zfs_zpool_* is largely empty. Everything else comes from the zpool/zfs CLIs
	# via ./collectors/zfs-metrics.py, scraped through the textfile collector.
	#
	# The wrapper exists only to put the matching zfs userland on PATH; the collector
	# itself is a plain script so it can be run and tested by hand.
	snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ];

	retentionArgs = lib.optionals config.services.zfs.autoSnapshot.enable (
		lib.concatMap (label: [
			"--retention"
			"${label}=${toString config.services.zfs.autoSnapshot.${label}}"
		]) snapshotLabels
	);

	zfsCollector = pkgs.writeShellApplication {
		name = "zfs-textfile-collector";
		runtimeInputs = [ config.boot.zfs.package ];
		text = ''
			exec ${pkgs.python3}/bin/python3 ${./collectors/zfs-metrics.py} \
				--output ${textfileDir}/zfs.prom ${lib.escapeShellArgs retentionArgs}
		'';
	};

	# smartctl_exporter reads only the entry *count* out of each drive's self-test log
	# and discards the log table, so it cannot say when a drive last ran a short or
	# extended test, or whether it passed. This parses `smartctl -j -l selftest`
	# directly. Device labels are bare names ("sda") so they join against the
	# exporter's own series.
	smartSelftestCollector = pkgs.writeShellApplication {
		name = "smart-selftest-collector";
		runtimeInputs = [ pkgs.smartmontools ];
		text = ''
			exec ${pkgs.python3}/bin/python3 ${./collectors/smart-selftest.py} \
				--output ${textfileDir}/smart-selftest.prom
		'';
	};
in
{
	##########################################################################
	# Host: log shipping
	##########################################################################

	services.alloy.enable = true;

	environment.etc."alloy/config.alloy".text = ''
		loki.source.journal "journal" {
			forward_to    = [loki.write.default.receiver]
			relabel_rules = loki.relabel.journal.rules
			max_age       = "12h"
		}

		loki.relabel "journal" {
			forward_to = []
			rule {
				source_labels = ["__journal__systemd_unit"]
				target_label  = "unit"
			}
			rule {
				source_labels = ["__journal__hostname"]
				target_label  = "host"
			}
			rule {
				source_labels = ["__journal_priority_keyword"]
				target_label  = "level"
			}
		}

		loki.write "default" {
			endpoint {
				url = "http://lgtm:${toString ports.loki}/loki/api/v1/push"
			}
		}
	'';

	##########################################################################
	# Host: telemetry sources
	#
	# These have to live on the host, not in the container: they read host
	# hardware and kernel state (/proc, /sys, SMART, ZFS ioctls) that a
	# container has no view of. Only the web services (Grafana, Prometheus,
	# Loki) run in the container.
	##########################################################################

	services.prometheus.exporters.node = {
		enable = true;
		port = ports.node;
		listenAddress = "0.0.0.0";

		# NOTE: enabledCollectors *adds* to node_exporter's defaults — cpu, diskstats,
		# filesystem, meminfo, zfs, hwmon, pressure and friends are already on, so
		# listing them here did nothing. Only genuinely opt-in collectors belong here.
		enabledCollectors = [
			"systemd"    # node_systemd_* — the "Systemd" row
			"processes"  # node_processes_* — "Processes Detailed States", "PIDs", "Threads"
			# "interrupts"  # fills the "IRQ Detail" panel, but emits one series per
			#               # (cpu, irq) pair — thousands of series on a many-core box.
			#               # Enable only if you want that one panel badly enough.
		];

		extraFlags = [
			"--collector.textfile.directory=${textfileDir}"
			# node_systemd_unit_start_time_seconds, so the dashboards can show when the
			# zfs-snapshot-*/zfs-scrub units last ran rather than only whether they failed.
			"--collector.systemd.enable-start-time-metrics"
		];
	};

	# Upstream hardens the unit with ProtectHome=true, which makes /home look empty and
	# blanks out the filesystem panels for any dataset mounted under it. read-only still
	# allows statfs while keeping the sandbox.
	systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only";

	services.prometheus.exporters.smartctl = {
		enable = true;
		port = ports.smartctl;
		listenAddress = "0.0.0.0";
		# Disks are direct-attached (AHCI/HBA), so auto-discovery via `smartctl --scan`
		# finds them; no explicit `devices` list needed. Throttle how often we actually
		# shell out to smartctl — SMART data changes slowly and polling spins up disks.
		maxInterval = "60s";
	};

	# SMART self-test history. Reading the log does not spin an idle disk up, but the
	# data only changes when a test finishes, so this runs far less often than the
	# exporter scrapes.
	systemd.services.smart-selftest-collector = {
		description = "Collect SMART self-test history for the Prometheus node exporter";
		serviceConfig = {
			Type = "oneshot";
			ExecStart = lib.getExe smartSelftestCollector;
			User = "root";
			StateDirectory = "prometheus-node-exporter-textfile";
			StateDirectoryMode = "0755";
			UMask = "0022";
			TimeoutStartSec = "180s";

			# Same shape as the smartctl exporter's own hardening: it needs raw device
			# access to talk to the disks, and nothing else.
			AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
			CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
			DevicePolicy = "closed";
			DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ];
			LockPersonality = true;
			MemoryDenyWriteExecute = true;
			NoNewPrivileges = true;
			PrivateNetwork = true;
			PrivateTmp = true;
			ProtectControlGroups = true;
			ProtectHome = true;
			ProtectHostname = true;
			ProtectKernelLogs = true;
			ProtectKernelModules = true;
			ProtectSystem = "strict";
			RestrictAddressFamilies = [ "AF_UNIX" ];
			RestrictNamespaces = true;
			RestrictRealtime = true;
			RestrictSUIDSGID = true;
			SystemCallArchitectures = "native";
		};
	};

	systemd.timers.smart-selftest-collector = {
		description = "Collect SMART self-test history periodically";
		wantedBy = [ "timers.target" ];
		timerConfig = {
			OnBootSec = "3min";
			OnUnitActiveSec = "15min";
			AccuracySec = "1min";
			Unit = "smart-selftest-collector.service";
		};
	};

	# Pool capacity and health, vdev errors, scrub history, dataset properties and
	# snapshot retention. See zfsCollector above for why node_exporter's built-in zfs
	# collector isn't enough on its own.
	systemd.services.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled {
		description = "Collect ZFS metrics for the Prometheus node exporter";
		serviceConfig = {
			Type = "oneshot";
			ExecStart = lib.getExe zfsCollector;
			User = "root";
			StateDirectory = "prometheus-node-exporter-textfile";
			StateDirectoryMode = "0755";
			UMask = "0022";  # node_exporter runs as a different user and must read these
			# Listing every snapshot on a busy pool is the slow part; keep this well
			# above the subprocess timeout inside the collector.
			TimeoutStartSec = "180s";

			# Hardening — this only needs to read ZFS state and write one file.
			# CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated
			# on it; everything else is dropped.
			CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ];
			LockPersonality = true;
			MemoryDenyWriteExecute = true;
			NoNewPrivileges = true;
			PrivateNetwork = true;
			PrivateTmp = true;
			ProtectControlGroups = true;
			ProtectHome = true;
			ProtectHostname = true;
			ProtectKernelLogs = true;
			ProtectKernelModules = true;
			ProtectSystem = "strict";
			RestrictAddressFamilies = [ "AF_UNIX" ];
			RestrictNamespaces = true;
			RestrictRealtime = true;
			RestrictSUIDSGID = true;
			SystemCallArchitectures = "native";
			SystemCallFilter = [ "@system-service" ];
		};
	};

	systemd.timers.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled {
		description = "Collect ZFS metrics periodically";
		wantedBy = [ "timers.target" ];
		timerConfig = {
			OnBootSec = "1min";
			OnUnitActiveSec = "1min";
			AccuracySec = "5s";
			Unit = "zfs-textfile-collector.service";
		};
	};

	# The collector used to write zpool.prom. Nothing removes a stale textfile once its
	# writer is gone, and node_exporter would happily keep serving those frozen metrics
	# forever, so delete it on activation.
	systemd.tmpfiles.rules = lib.mkIf config.boot.zfs.enabled [
		"r ${textfileDir}/zpool.prom"
	];

	# Reachable from the container, which sits on br-lan0 with its own address.
	# (Port 9273 is gone — that was telegraf/zpool_influxdb, replaced by the textfile
	# collector above, which needs no listener of its own.)
	networking.firewall.allowedTCPPorts = [ ports.node ports.smartctl ];

	##########################################################################
	# Secrets
	##########################################################################

	sops.secrets.grafana-secret-key = {
		sopsFile = ./resources/secrets/grafana.yaml;
		key = "secret_key";
	};

	##########################################################################
	# Container: Grafana / Prometheus / Loki
	##########################################################################

	containers.grafana = {
		autoStart = true;
		privateNetwork = true;
		hostBridge = "br-lan0";
		localMacAddress = "02:00:00:00:00:06";

		bindMounts."/run/secrets/grafana-secret_key" = {
			hostPath = config.sops.secrets.grafana-secret-key.path;
			isReadOnly = true;
		};

		config = { lib, pkgs, config, ... }: {
			# Network
			networking.enableIPv6 = false;
			networking.interfaces.eth0.useDHCP = true;
			networking.firewall.allowedTCPPorts = [
				80                 # Caddy -> Grafana
				ports.loki         # host's Alloy pushes here
				ports.prometheus   # handy for debugging scrape health directly
			];

			# ---------------------------------------------------------------
			# Loki
			# ---------------------------------------------------------------
			services.loki = {
				enable = true;
				configuration = {
					server = {
						http_listen_port = ports.loki;
						# Explicit, since the host's Alloy needs to reach it to push logs.
						http_listen_address = "0.0.0.0";
					};
					auth_enabled = false;

					common = {
						path_prefix = "/var/lib/loki";
						storage.filesystem = {
							chunks_directory = "/var/lib/loki/chunks";
							rules_directory = "/var/lib/loki/rules";
						};
						ring = {
							instance_addr = "127.0.0.1";
							kvstore.store = "inmemory";
						};
						replication_factor = 1;
					};

					schema_config.configs = [{
						from = "2024-01-01";
						store = "tsdb";
						object_store = "filesystem";
						schema = "v13";
						index = { prefix = "index_"; period = "24h"; };
					}];
				};
			};

			# ---------------------------------------------------------------
			# Prometheus
			# ---------------------------------------------------------------
			services.prometheus = {
				enable = true;

				# Without this Prometheus falls back to its built-in 1m scrape interval,
				# which desynchronises from what Grafana assumes and breaks every
				# rate()-based panel. See the comment on `scrapeInterval` at the top.
				globalConfig = {
					scrape_interval = scrapeInterval;
					evaluation_interval = scrapeInterval;
					scrape_timeout = "10s";
				};

				# Drive health verdict. Kept as recording rules rather than inlined in
				# dashboard panels so the logic is written once and reviewable on its
				# own. services.prometheus.checkConfig runs promtool at build time, so
				# a broken rule fails the build instead of the service.
				ruleFiles = [ ./rules/smartctl.yml ];

				scrapeConfigs = [
					{
						job_name = "node";
						static_configs = [{ targets = [ "${hostName}:${toString ports.node}" ]; }];
					}
					{
						job_name = "smartctl";
						# Querying SMART is comparatively slow and the data barely moves, so
						# this job opts out of the global 15s. Safe to do here because the
						# smartctl dashboard uses no rate() queries at all.
						scrape_interval = "60s";
						scrape_timeout = "30s";
						static_configs = [{ targets = [ "${hostName}:${toString ports.smartctl}" ]; }];
					}
				];
			};

			# ---------------------------------------------------------------
			# Grafana
			# https://wiki.nixos.org/wiki/Grafana
			# ---------------------------------------------------------------
			services.grafana = {
				enable = true;
				settings.server = { http_addr = "0.0.0.0"; http_port = ports.grafana; };

				provision = {
					enable = true;

					# Mutable dashboard provider pulling from /etc/grafana-dashboards, so
					# dashboards can be declared with environment.etc below.
					dashboards.settings.providers = [{
						name = "my dashboards";
						disableDeletion = true;
						options = {
							path = "/etc/grafana-dashboards";
							foldersFromFilesStructure = true;
						};
					}];

					# Grafana cannot change the UID of an existing datasource in place. Its
					# provisioner finds the datasource by name, then calls UpdateDataSource,
					# which re-fetches using `id AND uid` — with a new uid that row doesn't
					# match, so it fails with "data source not found" and Grafana refuses to
					# start. Deleting first turns the upsert into a plain insert; the
					# provisioner runs deletes before inserts for exactly this case, and
					# suppresses the deletion event because the datasource is about to be
					# re-created. Safe to leave in permanently: both datasources are fully
					# declarative, and dashboards bind to them by UID, not by database ID.
					datasources.settings.deleteDatasources = [
						{ name = "Prometheus"; orgId = 1; }
						{ name = "Loki"; orgId = 1; }
					];

					datasources.settings.datasources = [
						{
							name = "Prometheus";
							type = "prometheus";
							uid = dsUid.prometheus;  # fixed, so dashboards can reference it
							url = "http://127.0.0.1:${toString ports.prometheus}";
							isDefault = true;
							jsonData = {
								# Tells Grafana how often the data is actually scraped. This is
								# what $__rate_interval is derived from; leaving it unset makes
								# Grafana guess 15s and mis-size every rate() window.
								timeInterval = scrapeInterval;
								httpMethod = "POST";
							};
						}
						{
							name = "Loki";
							type = "loki";
							uid = dsUid.loki;
							url = "http://127.0.0.1:${toString ports.loki}";
						}
					];
				};
			};

			# Grafana secret key
			systemd.services.grafana.serviceConfig.LoadCredential = [
				"secret_key:/run/secrets/grafana-secret_key"
			];
			services.grafana.settings.security.secret_key =
				"$__file{/run/credentials/grafana.service/secret_key}";

			# ---------------------------------------------------------------
			# Dashboards
			# ---------------------------------------------------------------
			environment.etc = lib.mapAttrs' (name: drv:
				lib.nameValuePair "grafana-dashboards/${name}.json" { source = drv; }
			) dashboards;

			# ---------------------------------------------------------------
			# Reverse proxy
			# ---------------------------------------------------------------
			services.caddy = {
				enable = true;
				virtualHosts.":80".extraConfig = ''
					reverse_proxy localhost:${toString ports.grafana}
				'';
			};

			system.stateVersion = "26.05";
		};
	};
}