diff options
Diffstat (limited to 'lgtm.nix')
| -rw-r--r-- | lgtm.nix | 534 |
1 files changed, 534 insertions, 0 deletions
diff --git a/lgtm.nix b/lgtm.nix new file mode 100644 index 0000000..a4144ec --- /dev/null +++ b/lgtm.nix @@ -0,0 +1,534 @@ +{ pkgs, config, lib, ... }: + +let + # The hostname the container uses to reach the host's exporters. + hostName = "poweredge"; + + # One scrape interval to rule them all. This MUST match the Grafana Prometheus + # datasource's `timeInterval` below, otherwise Grafana computes $__rate_interval + # from the wrong assumption and every rate()-based panel intermittently renders + # "No data". (Prometheus' own default is 1m, which is what broke the Node Exporter + # Full dashboard: $__rate_interval resolved to 60s, so rate(x[60s]) over 60s-spaced + # samples usually saw fewer than the two samples rate() requires.) + scrapeInterval = "15s"; + + # Where the host's zpool collector drops its metrics for node_exporter to pick up. + textfileDir = "/var/lib/prometheus-node-exporter-textfile"; + + ports = { + node = 9100; + smartctl = 9633; + loki = 3100; + prometheus = 9090; + grafana = 3000; + }; + + # Stable datasource UIDs, referenced by the dashboards below. + dsUid = { prometheus = "prometheus"; loki = "loki"; }; + + ########################################################################## + # Dashboard plumbing + ########################################################################## + + # Dashboards exported from grafana.com carry an `__inputs` block and reference + # their datasource as "${DS_PROMETHEUS}". That placeholder is only substituted by + # Grafana's interactive *import* flow — file-based provisioning never runs it, so + # the uid stays literal and every panel fails to resolve a datasource. This filter + # performs the substitution at build time, using our fixed datasource UIDs. + dashboardFixup = pkgs.writeText "grafana-dashboard-fixup.jq" '' + def dsuid($pluginId): + if $pluginId == "prometheus" then $prom + elif $pluginId == "loki" then $loki + else "" end; + + def subst($name; $uid): + walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end); + + reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId))) + | del(.__inputs, .__requires) + # Provisioned dashboards must not carry an id, or Grafana rejects them on + # collision with whatever already occupies that id in its database. + | .id = null + ''; + + fixupDashboard = name: src: pkgs.runCommand "grafana-dashboard-${name}.json" + { nativeBuildInputs = [ pkgs.jq ]; } + '' + jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \ + --arg loki ${lib.escapeShellArg dsUid.loki} \ + -f ${dashboardFixup} < ${src} > "$out" + ''; + + # grafana.com dashboard, pinned by revision. + mkGrafanaComDashboard = { name, id, rev, sha256 }: + fixupDashboard name (pkgs.fetchurl { + url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download"; + inherit sha256; + }); + + dashboards = { + # https://grafana.com/grafana/dashboards/1860 — Node Exporter Full + node-exporter-full = mkGrafanaComDashboard { + name = "node-exporter-full"; id = 1860; rev = 45; + sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY="; + }; + + # https://grafana.com/grafana/dashboards/22604 — Smartctl Exporter + smartctl-exporter = mkGrafanaComDashboard { + name = "smartctl-exporter"; id = 22604; rev = 2; + sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88="; + }; + + # Dashboard 15362 ("ZFS Pool Metrics") used to live here. It is an *InfluxDB* + # dashboard — its queries are InfluxQL against a `zpool_stats` measurement — so + # it can never render against Prometheus, no matter how it is provisioned. This + # is a Prometheus-native replacement built on the metrics we actually export: + # node_exporter's zfs collector plus the zpool textfile collector below. + zfs-pools = fixupDashboard "zfs-pools" ./dashboards/zfs-pools.json; + + # CrystalDiskInfo-style per-drive SMART view. Generated by + # ./dashboards/gen_smart.py; the Good/Caution/Bad verdict it displays comes + # from the recording rules in ./rules/smartctl.yml. + smart = fixupDashboard "smart" ./dashboards/smart.json; + }; + + ########################################################################## + # ZFS metrics (host) + ########################################################################## + + # node_exporter's zfs collector reads /proc/spl/kstat/zfs, which gives us ARC + # statistics, per-dataset objset counters and pool state -- but nothing about + # capacity, fragmentation, vdev errors, scrub history, compression, encryption or + # snapshots. OpenZFS 2.2+ also no longer publishes the per-pool `io` kstat, so + # node_zfs_zpool_* is largely empty. Everything else comes from the zpool/zfs CLIs + # via ./collectors/zfs-metrics.py, scraped through the textfile collector. + # + # The wrapper exists only to put the matching zfs userland on PATH; the collector + # itself is a plain script so it can be run and tested by hand. + snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ]; + + retentionArgs = lib.optionals config.services.zfs.autoSnapshot.enable ( + lib.concatMap (label: [ + "--retention" + "${label}=${toString config.services.zfs.autoSnapshot.${label}}" + ]) snapshotLabels + ); + + zfsCollector = pkgs.writeShellApplication { + name = "zfs-textfile-collector"; + runtimeInputs = [ config.boot.zfs.package ]; + text = '' + exec ${pkgs.python3}/bin/python3 ${./collectors/zfs-metrics.py} \ + --output ${textfileDir}/zfs.prom ${lib.escapeShellArgs retentionArgs} + ''; + }; + + # smartctl_exporter reads only the entry *count* out of each drive's self-test log + # and discards the log table, so it cannot say when a drive last ran a short or + # extended test, or whether it passed. This parses `smartctl -j -l selftest` + # directly. Device labels are bare names ("sda") so they join against the + # exporter's own series. + smartSelftestCollector = pkgs.writeShellApplication { + name = "smart-selftest-collector"; + runtimeInputs = [ pkgs.smartmontools ]; + text = '' + exec ${pkgs.python3}/bin/python3 ${./collectors/smart-selftest.py} \ + --output ${textfileDir}/smart-selftest.prom + ''; + }; +in +{ + ########################################################################## + # Host: log shipping + ########################################################################## + + services.alloy.enable = true; + + environment.etc."alloy/config.alloy".text = '' + loki.source.journal "journal" { + forward_to = [loki.write.default.receiver] + relabel_rules = loki.relabel.journal.rules + max_age = "12h" + } + + loki.relabel "journal" { + forward_to = [] + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal__hostname"] + target_label = "host" + } + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + } + + loki.write "default" { + endpoint { + url = "http://lgtm:${toString ports.loki}/loki/api/v1/push" + } + } + ''; + + ########################################################################## + # Host: telemetry sources + # + # These have to live on the host, not in the container: they read host + # hardware and kernel state (/proc, /sys, SMART, ZFS ioctls) that a + # container has no view of. Only the web services (Grafana, Prometheus, + # Loki) run in the container. + ########################################################################## + + services.prometheus.exporters.node = { + enable = true; + port = ports.node; + listenAddress = "0.0.0.0"; + + # NOTE: enabledCollectors *adds* to node_exporter's defaults — cpu, diskstats, + # filesystem, meminfo, zfs, hwmon, pressure and friends are already on, so + # listing them here did nothing. Only genuinely opt-in collectors belong here. + enabledCollectors = [ + "systemd" # node_systemd_* — the "Systemd" row + "processes" # node_processes_* — "Processes Detailed States", "PIDs", "Threads" + # "interrupts" # fills the "IRQ Detail" panel, but emits one series per + # # (cpu, irq) pair — thousands of series on a many-core box. + # # Enable only if you want that one panel badly enough. + ]; + + extraFlags = [ + "--collector.textfile.directory=${textfileDir}" + # node_systemd_unit_start_time_seconds, so the dashboards can show when the + # zfs-snapshot-*/zfs-scrub units last ran rather than only whether they failed. + "--collector.systemd.enable-start-time-metrics" + ]; + }; + + # Upstream hardens the unit with ProtectHome=true, which makes /home look empty and + # blanks out the filesystem panels for any dataset mounted under it. read-only still + # allows statfs while keeping the sandbox. + systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only"; + + services.prometheus.exporters.smartctl = { + enable = true; + port = ports.smartctl; + listenAddress = "0.0.0.0"; + # Disks are direct-attached (AHCI/HBA), so auto-discovery via `smartctl --scan` + # finds them; no explicit `devices` list needed. Throttle how often we actually + # shell out to smartctl — SMART data changes slowly and polling spins up disks. + maxInterval = "60s"; + }; + + # SMART self-test history. Reading the log does not spin an idle disk up, but the + # data only changes when a test finishes, so this runs far less often than the + # exporter scrapes. + systemd.services.smart-selftest-collector = { + description = "Collect SMART self-test history for the Prometheus node exporter"; + serviceConfig = { + Type = "oneshot"; + ExecStart = lib.getExe smartSelftestCollector; + User = "root"; + StateDirectory = "prometheus-node-exporter-textfile"; + StateDirectoryMode = "0755"; + UMask = "0022"; + TimeoutStartSec = "180s"; + + # Same shape as the smartctl exporter's own hardening: it needs raw device + # access to talk to the disks, and nothing else. + AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ]; + CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ]; + DevicePolicy = "closed"; + DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ]; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateNetwork = true; + PrivateTmp = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectSystem = "strict"; + RestrictAddressFamilies = [ "AF_UNIX" ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + }; + }; + + systemd.timers.smart-selftest-collector = { + description = "Collect SMART self-test history periodically"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = "3min"; + OnUnitActiveSec = "15min"; + AccuracySec = "1min"; + Unit = "smart-selftest-collector.service"; + }; + }; + + # Pool capacity and health, vdev errors, scrub history, dataset properties and + # snapshot retention. See zfsCollector above for why node_exporter's built-in zfs + # collector isn't enough on its own. + systemd.services.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled { + description = "Collect ZFS metrics for the Prometheus node exporter"; + serviceConfig = { + Type = "oneshot"; + ExecStart = lib.getExe zfsCollector; + User = "root"; + StateDirectory = "prometheus-node-exporter-textfile"; + StateDirectoryMode = "0755"; + UMask = "0022"; # node_exporter runs as a different user and must read these + # Listing every snapshot on a busy pool is the slow part; keep this well + # above the subprocess timeout inside the collector. + TimeoutStartSec = "180s"; + + # Hardening — this only needs to read ZFS state and write one file. + # CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated + # on it; everything else is dropped. + CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ]; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateNetwork = true; + PrivateTmp = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectSystem = "strict"; + RestrictAddressFamilies = [ "AF_UNIX" ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ "@system-service" ]; + }; + }; + + systemd.timers.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled { + description = "Collect ZFS metrics periodically"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = "1min"; + OnUnitActiveSec = "1min"; + AccuracySec = "5s"; + Unit = "zfs-textfile-collector.service"; + }; + }; + + # The collector used to write zpool.prom. Nothing removes a stale textfile once its + # writer is gone, and node_exporter would happily keep serving those frozen metrics + # forever, so delete it on activation. + systemd.tmpfiles.rules = lib.mkIf config.boot.zfs.enabled [ + "r ${textfileDir}/zpool.prom" + ]; + + # Reachable from the container, which sits on br-lan0 with its own address. + # (Port 9273 is gone — that was telegraf/zpool_influxdb, replaced by the textfile + # collector above, which needs no listener of its own.) + networking.firewall.allowedTCPPorts = [ ports.node ports.smartctl ]; + + ########################################################################## + # Secrets + ########################################################################## + + sops.secrets.grafana-secret-key = { + sopsFile = ./resources/secrets/grafana.yaml; + key = "secret_key"; + }; + + ########################################################################## + # Container: Grafana / Prometheus / Loki + ########################################################################## + + containers.grafana = { + autoStart = true; + privateNetwork = true; + hostBridge = "br-lan0"; + localMacAddress = "02:00:00:00:00:06"; + + bindMounts."/run/secrets/grafana-secret_key" = { + hostPath = config.sops.secrets.grafana-secret-key.path; + isReadOnly = true; + }; + + config = { lib, pkgs, config, ... }: { + # Network + networking.enableIPv6 = false; + networking.interfaces.eth0.useDHCP = true; + networking.firewall.allowedTCPPorts = [ + 80 # Caddy -> Grafana + ports.loki # host's Alloy pushes here + ports.prometheus # handy for debugging scrape health directly + ]; + + # --------------------------------------------------------------- + # Loki + # --------------------------------------------------------------- + services.loki = { + enable = true; + configuration = { + server = { + http_listen_port = ports.loki; + # Explicit, since the host's Alloy needs to reach it to push logs. + http_listen_address = "0.0.0.0"; + }; + auth_enabled = false; + + common = { + path_prefix = "/var/lib/loki"; + storage.filesystem = { + chunks_directory = "/var/lib/loki/chunks"; + rules_directory = "/var/lib/loki/rules"; + }; + ring = { + instance_addr = "127.0.0.1"; + kvstore.store = "inmemory"; + }; + replication_factor = 1; + }; + + schema_config.configs = [{ + from = "2024-01-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { prefix = "index_"; period = "24h"; }; + }]; + }; + }; + + # --------------------------------------------------------------- + # Prometheus + # --------------------------------------------------------------- + services.prometheus = { + enable = true; + + # Without this Prometheus falls back to its built-in 1m scrape interval, + # which desynchronises from what Grafana assumes and breaks every + # rate()-based panel. See the comment on `scrapeInterval` at the top. + globalConfig = { + scrape_interval = scrapeInterval; + evaluation_interval = scrapeInterval; + scrape_timeout = "10s"; + }; + + # Drive health verdict. Kept as recording rules rather than inlined in + # dashboard panels so the logic is written once and reviewable on its + # own. services.prometheus.checkConfig runs promtool at build time, so + # a broken rule fails the build instead of the service. + ruleFiles = [ ./rules/smartctl.yml ]; + + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [{ targets = [ "${hostName}:${toString ports.node}" ]; }]; + } + { + job_name = "smartctl"; + # Querying SMART is comparatively slow and the data barely moves, so + # this job opts out of the global 15s. Safe to do here because the + # smartctl dashboard uses no rate() queries at all. + scrape_interval = "60s"; + scrape_timeout = "30s"; + static_configs = [{ targets = [ "${hostName}:${toString ports.smartctl}" ]; }]; + } + ]; + }; + + # --------------------------------------------------------------- + # Grafana + # https://wiki.nixos.org/wiki/Grafana + # --------------------------------------------------------------- + services.grafana = { + enable = true; + settings.server = { http_addr = "0.0.0.0"; http_port = ports.grafana; }; + + provision = { + enable = true; + + # Mutable dashboard provider pulling from /etc/grafana-dashboards, so + # dashboards can be declared with environment.etc below. + dashboards.settings.providers = [{ + name = "my dashboards"; + disableDeletion = true; + options = { + path = "/etc/grafana-dashboards"; + foldersFromFilesStructure = true; + }; + }]; + + # Grafana cannot change the UID of an existing datasource in place. Its + # provisioner finds the datasource by name, then calls UpdateDataSource, + # which re-fetches using `id AND uid` — with a new uid that row doesn't + # match, so it fails with "data source not found" and Grafana refuses to + # start. Deleting first turns the upsert into a plain insert; the + # provisioner runs deletes before inserts for exactly this case, and + # suppresses the deletion event because the datasource is about to be + # re-created. Safe to leave in permanently: both datasources are fully + # declarative, and dashboards bind to them by UID, not by database ID. + datasources.settings.deleteDatasources = [ + { name = "Prometheus"; orgId = 1; } + { name = "Loki"; orgId = 1; } + ]; + + datasources.settings.datasources = [ + { + name = "Prometheus"; + type = "prometheus"; + uid = dsUid.prometheus; # fixed, so dashboards can reference it + url = "http://127.0.0.1:${toString ports.prometheus}"; + isDefault = true; + jsonData = { + # Tells Grafana how often the data is actually scraped. This is + # what $__rate_interval is derived from; leaving it unset makes + # Grafana guess 15s and mis-size every rate() window. + timeInterval = scrapeInterval; + httpMethod = "POST"; + }; + } + { + name = "Loki"; + type = "loki"; + uid = dsUid.loki; + url = "http://127.0.0.1:${toString ports.loki}"; + } + ]; + }; + }; + + # Grafana secret key + systemd.services.grafana.serviceConfig.LoadCredential = [ + "secret_key:/run/secrets/grafana-secret_key" + ]; + services.grafana.settings.security.secret_key = + "$__file{/run/credentials/grafana.service/secret_key}"; + + # --------------------------------------------------------------- + # Dashboards + # --------------------------------------------------------------- + environment.etc = lib.mapAttrs' (name: drv: + lib.nameValuePair "grafana-dashboards/${name}.json" { source = drv; } + ) dashboards; + + # --------------------------------------------------------------- + # Reverse proxy + # --------------------------------------------------------------- + services.caddy = { + enable = true; + virtualHosts.":80".extraConfig = '' + reverse_proxy localhost:${toString ports.grafana} + ''; + }; + + system.stateVersion = "26.05"; + }; + }; +} |