From d2d1386abb358fa547a40fc8df613d799155b56d Mon Sep 17 00:00:00 2001 From: Tim Keller Date: Thu, 27 Aug 2026 12:45:44 -0500 Subject: split into a flake with separable collectors/server modules Replaces the single lgtm.nix with a flake exposing two NixOS modules under services.telemetry, so the web services can move into a container declared in the consuming config rather than one hardcoded here. modules/collectors.nix host-side: node + smartctl exporters, the ZFS and SMART textfile collector timers, Alloy journal shipping. Must run on the host -- reads /proc, /sys, raw block devices and ZFS ioctls. modules/server.nix Prometheus, Loki, Grafana, optional Caddy. modules/common.nix ports, shared by both halves. modules/dashboards.nix jq fixup and grafana.com fetches. modules/packages.nix the two collectors as ordinary packages. Modules are exported as paths, not functions, so they evaluate with the consumer's nixpkgs; the flake's own input only affects packages/checks/devShells. Multi-host support. collectorHosts is a list, both scrape jobs relabel __address__ into a `host` label, and all 133 queries in the two generated dashboards are scoped to host=~"$host" behind a single-select Host picker. `instance` could not serve as that key: one machine is scraped twice on different ports (textfile metrics on :9100, smartctl_* on :9633) and the recording rules join across that boundary. Grafana's secret key becomes a plain secretKeyFile path wired through LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No sops-nix input. nix flake check runs promtool over the rules, diffs the committed dashboard JSON against what the generators produce, and evaluates both modules. Also fixes smart-selftest emitting a placeholder identity line when smartctl cannot open a device. smartctl exits 0 with well-formed but empty JSON in that case, so "SSD - unknown model - no serial" was reaching a dashboard row title; it now emits nothing and lets the smartctl:ident fallback apply. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ --- lgtm.nix | 534 --------------------------------------------------------------- 1 file changed, 534 deletions(-) delete mode 100644 lgtm.nix (limited to 'lgtm.nix') diff --git a/lgtm.nix b/lgtm.nix deleted file mode 100644 index a4144ec..0000000 --- a/lgtm.nix +++ /dev/null @@ -1,534 +0,0 @@ -{ pkgs, config, lib, ... }: - -let - # The hostname the container uses to reach the host's exporters. - hostName = "poweredge"; - - # One scrape interval to rule them all. This MUST match the Grafana Prometheus - # datasource's `timeInterval` below, otherwise Grafana computes $__rate_interval - # from the wrong assumption and every rate()-based panel intermittently renders - # "No data". (Prometheus' own default is 1m, which is what broke the Node Exporter - # Full dashboard: $__rate_interval resolved to 60s, so rate(x[60s]) over 60s-spaced - # samples usually saw fewer than the two samples rate() requires.) - scrapeInterval = "15s"; - - # Where the host's zpool collector drops its metrics for node_exporter to pick up. - textfileDir = "/var/lib/prometheus-node-exporter-textfile"; - - ports = { - node = 9100; - smartctl = 9633; - loki = 3100; - prometheus = 9090; - grafana = 3000; - }; - - # Stable datasource UIDs, referenced by the dashboards below. - dsUid = { prometheus = "prometheus"; loki = "loki"; }; - - ########################################################################## - # Dashboard plumbing - ########################################################################## - - # Dashboards exported from grafana.com carry an `__inputs` block and reference - # their datasource as "${DS_PROMETHEUS}". That placeholder is only substituted by - # Grafana's interactive *import* flow — file-based provisioning never runs it, so - # the uid stays literal and every panel fails to resolve a datasource. This filter - # performs the substitution at build time, using our fixed datasource UIDs. - dashboardFixup = pkgs.writeText "grafana-dashboard-fixup.jq" '' - def dsuid($pluginId): - if $pluginId == "prometheus" then $prom - elif $pluginId == "loki" then $loki - else "" end; - - def subst($name; $uid): - walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end); - - reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId))) - | del(.__inputs, .__requires) - # Provisioned dashboards must not carry an id, or Grafana rejects them on - # collision with whatever already occupies that id in its database. - | .id = null - ''; - - fixupDashboard = name: src: pkgs.runCommand "grafana-dashboard-${name}.json" - { nativeBuildInputs = [ pkgs.jq ]; } - '' - jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \ - --arg loki ${lib.escapeShellArg dsUid.loki} \ - -f ${dashboardFixup} < ${src} > "$out" - ''; - - # grafana.com dashboard, pinned by revision. - mkGrafanaComDashboard = { name, id, rev, sha256 }: - fixupDashboard name (pkgs.fetchurl { - url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download"; - inherit sha256; - }); - - dashboards = { - # https://grafana.com/grafana/dashboards/1860 — Node Exporter Full - node-exporter-full = mkGrafanaComDashboard { - name = "node-exporter-full"; id = 1860; rev = 45; - sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY="; - }; - - # https://grafana.com/grafana/dashboards/22604 — Smartctl Exporter - smartctl-exporter = mkGrafanaComDashboard { - name = "smartctl-exporter"; id = 22604; rev = 2; - sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88="; - }; - - # Dashboard 15362 ("ZFS Pool Metrics") used to live here. It is an *InfluxDB* - # dashboard — its queries are InfluxQL against a `zpool_stats` measurement — so - # it can never render against Prometheus, no matter how it is provisioned. This - # is a Prometheus-native replacement built on the metrics we actually export: - # node_exporter's zfs collector plus the zpool textfile collector below. - zfs-pools = fixupDashboard "zfs-pools" ./dashboards/zfs-pools.json; - - # CrystalDiskInfo-style per-drive SMART view. Generated by - # ./dashboards/gen_smart.py; the Good/Caution/Bad verdict it displays comes - # from the recording rules in ./rules/smartctl.yml. - smart = fixupDashboard "smart" ./dashboards/smart.json; - }; - - ########################################################################## - # ZFS metrics (host) - ########################################################################## - - # node_exporter's zfs collector reads /proc/spl/kstat/zfs, which gives us ARC - # statistics, per-dataset objset counters and pool state -- but nothing about - # capacity, fragmentation, vdev errors, scrub history, compression, encryption or - # snapshots. OpenZFS 2.2+ also no longer publishes the per-pool `io` kstat, so - # node_zfs_zpool_* is largely empty. Everything else comes from the zpool/zfs CLIs - # via ./collectors/zfs-metrics.py, scraped through the textfile collector. - # - # The wrapper exists only to put the matching zfs userland on PATH; the collector - # itself is a plain script so it can be run and tested by hand. - snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ]; - - retentionArgs = lib.optionals config.services.zfs.autoSnapshot.enable ( - lib.concatMap (label: [ - "--retention" - "${label}=${toString config.services.zfs.autoSnapshot.${label}}" - ]) snapshotLabels - ); - - zfsCollector = pkgs.writeShellApplication { - name = "zfs-textfile-collector"; - runtimeInputs = [ config.boot.zfs.package ]; - text = '' - exec ${pkgs.python3}/bin/python3 ${./collectors/zfs-metrics.py} \ - --output ${textfileDir}/zfs.prom ${lib.escapeShellArgs retentionArgs} - ''; - }; - - # smartctl_exporter reads only the entry *count* out of each drive's self-test log - # and discards the log table, so it cannot say when a drive last ran a short or - # extended test, or whether it passed. This parses `smartctl -j -l selftest` - # directly. Device labels are bare names ("sda") so they join against the - # exporter's own series. - smartSelftestCollector = pkgs.writeShellApplication { - name = "smart-selftest-collector"; - runtimeInputs = [ pkgs.smartmontools ]; - text = '' - exec ${pkgs.python3}/bin/python3 ${./collectors/smart-selftest.py} \ - --output ${textfileDir}/smart-selftest.prom - ''; - }; -in -{ - ########################################################################## - # Host: log shipping - ########################################################################## - - services.alloy.enable = true; - - environment.etc."alloy/config.alloy".text = '' - loki.source.journal "journal" { - forward_to = [loki.write.default.receiver] - relabel_rules = loki.relabel.journal.rules - max_age = "12h" - } - - loki.relabel "journal" { - forward_to = [] - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "unit" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "host" - } - rule { - source_labels = ["__journal_priority_keyword"] - target_label = "level" - } - } - - loki.write "default" { - endpoint { - url = "http://lgtm:${toString ports.loki}/loki/api/v1/push" - } - } - ''; - - ########################################################################## - # Host: telemetry sources - # - # These have to live on the host, not in the container: they read host - # hardware and kernel state (/proc, /sys, SMART, ZFS ioctls) that a - # container has no view of. Only the web services (Grafana, Prometheus, - # Loki) run in the container. - ########################################################################## - - services.prometheus.exporters.node = { - enable = true; - port = ports.node; - listenAddress = "0.0.0.0"; - - # NOTE: enabledCollectors *adds* to node_exporter's defaults — cpu, diskstats, - # filesystem, meminfo, zfs, hwmon, pressure and friends are already on, so - # listing them here did nothing. Only genuinely opt-in collectors belong here. - enabledCollectors = [ - "systemd" # node_systemd_* — the "Systemd" row - "processes" # node_processes_* — "Processes Detailed States", "PIDs", "Threads" - # "interrupts" # fills the "IRQ Detail" panel, but emits one series per - # # (cpu, irq) pair — thousands of series on a many-core box. - # # Enable only if you want that one panel badly enough. - ]; - - extraFlags = [ - "--collector.textfile.directory=${textfileDir}" - # node_systemd_unit_start_time_seconds, so the dashboards can show when the - # zfs-snapshot-*/zfs-scrub units last ran rather than only whether they failed. - "--collector.systemd.enable-start-time-metrics" - ]; - }; - - # Upstream hardens the unit with ProtectHome=true, which makes /home look empty and - # blanks out the filesystem panels for any dataset mounted under it. read-only still - # allows statfs while keeping the sandbox. - systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only"; - - services.prometheus.exporters.smartctl = { - enable = true; - port = ports.smartctl; - listenAddress = "0.0.0.0"; - # Disks are direct-attached (AHCI/HBA), so auto-discovery via `smartctl --scan` - # finds them; no explicit `devices` list needed. Throttle how often we actually - # shell out to smartctl — SMART data changes slowly and polling spins up disks. - maxInterval = "60s"; - }; - - # SMART self-test history. Reading the log does not spin an idle disk up, but the - # data only changes when a test finishes, so this runs far less often than the - # exporter scrapes. - systemd.services.smart-selftest-collector = { - description = "Collect SMART self-test history for the Prometheus node exporter"; - serviceConfig = { - Type = "oneshot"; - ExecStart = lib.getExe smartSelftestCollector; - User = "root"; - StateDirectory = "prometheus-node-exporter-textfile"; - StateDirectoryMode = "0755"; - UMask = "0022"; - TimeoutStartSec = "180s"; - - # Same shape as the smartctl exporter's own hardening: it needs raw device - # access to talk to the disks, and nothing else. - AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ]; - CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ]; - DevicePolicy = "closed"; - DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ]; - LockPersonality = true; - MemoryDenyWriteExecute = true; - NoNewPrivileges = true; - PrivateNetwork = true; - PrivateTmp = true; - ProtectControlGroups = true; - ProtectHome = true; - ProtectHostname = true; - ProtectKernelLogs = true; - ProtectKernelModules = true; - ProtectSystem = "strict"; - RestrictAddressFamilies = [ "AF_UNIX" ]; - RestrictNamespaces = true; - RestrictRealtime = true; - RestrictSUIDSGID = true; - SystemCallArchitectures = "native"; - }; - }; - - systemd.timers.smart-selftest-collector = { - description = "Collect SMART self-test history periodically"; - wantedBy = [ "timers.target" ]; - timerConfig = { - OnBootSec = "3min"; - OnUnitActiveSec = "15min"; - AccuracySec = "1min"; - Unit = "smart-selftest-collector.service"; - }; - }; - - # Pool capacity and health, vdev errors, scrub history, dataset properties and - # snapshot retention. See zfsCollector above for why node_exporter's built-in zfs - # collector isn't enough on its own. - systemd.services.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled { - description = "Collect ZFS metrics for the Prometheus node exporter"; - serviceConfig = { - Type = "oneshot"; - ExecStart = lib.getExe zfsCollector; - User = "root"; - StateDirectory = "prometheus-node-exporter-textfile"; - StateDirectoryMode = "0755"; - UMask = "0022"; # node_exporter runs as a different user and must read these - # Listing every snapshot on a busy pool is the slow part; keep this well - # above the subprocess timeout inside the collector. - TimeoutStartSec = "180s"; - - # Hardening — this only needs to read ZFS state and write one file. - # CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated - # on it; everything else is dropped. - CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ]; - LockPersonality = true; - MemoryDenyWriteExecute = true; - NoNewPrivileges = true; - PrivateNetwork = true; - PrivateTmp = true; - ProtectControlGroups = true; - ProtectHome = true; - ProtectHostname = true; - ProtectKernelLogs = true; - ProtectKernelModules = true; - ProtectSystem = "strict"; - RestrictAddressFamilies = [ "AF_UNIX" ]; - RestrictNamespaces = true; - RestrictRealtime = true; - RestrictSUIDSGID = true; - SystemCallArchitectures = "native"; - SystemCallFilter = [ "@system-service" ]; - }; - }; - - systemd.timers.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled { - description = "Collect ZFS metrics periodically"; - wantedBy = [ "timers.target" ]; - timerConfig = { - OnBootSec = "1min"; - OnUnitActiveSec = "1min"; - AccuracySec = "5s"; - Unit = "zfs-textfile-collector.service"; - }; - }; - - # The collector used to write zpool.prom. Nothing removes a stale textfile once its - # writer is gone, and node_exporter would happily keep serving those frozen metrics - # forever, so delete it on activation. - systemd.tmpfiles.rules = lib.mkIf config.boot.zfs.enabled [ - "r ${textfileDir}/zpool.prom" - ]; - - # Reachable from the container, which sits on br-lan0 with its own address. - # (Port 9273 is gone — that was telegraf/zpool_influxdb, replaced by the textfile - # collector above, which needs no listener of its own.) - networking.firewall.allowedTCPPorts = [ ports.node ports.smartctl ]; - - ########################################################################## - # Secrets - ########################################################################## - - sops.secrets.grafana-secret-key = { - sopsFile = ./resources/secrets/grafana.yaml; - key = "secret_key"; - }; - - ########################################################################## - # Container: Grafana / Prometheus / Loki - ########################################################################## - - containers.grafana = { - autoStart = true; - privateNetwork = true; - hostBridge = "br-lan0"; - localMacAddress = "02:00:00:00:00:06"; - - bindMounts."/run/secrets/grafana-secret_key" = { - hostPath = config.sops.secrets.grafana-secret-key.path; - isReadOnly = true; - }; - - config = { lib, pkgs, config, ... }: { - # Network - networking.enableIPv6 = false; - networking.interfaces.eth0.useDHCP = true; - networking.firewall.allowedTCPPorts = [ - 80 # Caddy -> Grafana - ports.loki # host's Alloy pushes here - ports.prometheus # handy for debugging scrape health directly - ]; - - # --------------------------------------------------------------- - # Loki - # --------------------------------------------------------------- - services.loki = { - enable = true; - configuration = { - server = { - http_listen_port = ports.loki; - # Explicit, since the host's Alloy needs to reach it to push logs. - http_listen_address = "0.0.0.0"; - }; - auth_enabled = false; - - common = { - path_prefix = "/var/lib/loki"; - storage.filesystem = { - chunks_directory = "/var/lib/loki/chunks"; - rules_directory = "/var/lib/loki/rules"; - }; - ring = { - instance_addr = "127.0.0.1"; - kvstore.store = "inmemory"; - }; - replication_factor = 1; - }; - - schema_config.configs = [{ - from = "2024-01-01"; - store = "tsdb"; - object_store = "filesystem"; - schema = "v13"; - index = { prefix = "index_"; period = "24h"; }; - }]; - }; - }; - - # --------------------------------------------------------------- - # Prometheus - # --------------------------------------------------------------- - services.prometheus = { - enable = true; - - # Without this Prometheus falls back to its built-in 1m scrape interval, - # which desynchronises from what Grafana assumes and breaks every - # rate()-based panel. See the comment on `scrapeInterval` at the top. - globalConfig = { - scrape_interval = scrapeInterval; - evaluation_interval = scrapeInterval; - scrape_timeout = "10s"; - }; - - # Drive health verdict. Kept as recording rules rather than inlined in - # dashboard panels so the logic is written once and reviewable on its - # own. services.prometheus.checkConfig runs promtool at build time, so - # a broken rule fails the build instead of the service. - ruleFiles = [ ./rules/smartctl.yml ]; - - scrapeConfigs = [ - { - job_name = "node"; - static_configs = [{ targets = [ "${hostName}:${toString ports.node}" ]; }]; - } - { - job_name = "smartctl"; - # Querying SMART is comparatively slow and the data barely moves, so - # this job opts out of the global 15s. Safe to do here because the - # smartctl dashboard uses no rate() queries at all. - scrape_interval = "60s"; - scrape_timeout = "30s"; - static_configs = [{ targets = [ "${hostName}:${toString ports.smartctl}" ]; }]; - } - ]; - }; - - # --------------------------------------------------------------- - # Grafana - # https://wiki.nixos.org/wiki/Grafana - # --------------------------------------------------------------- - services.grafana = { - enable = true; - settings.server = { http_addr = "0.0.0.0"; http_port = ports.grafana; }; - - provision = { - enable = true; - - # Mutable dashboard provider pulling from /etc/grafana-dashboards, so - # dashboards can be declared with environment.etc below. - dashboards.settings.providers = [{ - name = "my dashboards"; - disableDeletion = true; - options = { - path = "/etc/grafana-dashboards"; - foldersFromFilesStructure = true; - }; - }]; - - # Grafana cannot change the UID of an existing datasource in place. Its - # provisioner finds the datasource by name, then calls UpdateDataSource, - # which re-fetches using `id AND uid` — with a new uid that row doesn't - # match, so it fails with "data source not found" and Grafana refuses to - # start. Deleting first turns the upsert into a plain insert; the - # provisioner runs deletes before inserts for exactly this case, and - # suppresses the deletion event because the datasource is about to be - # re-created. Safe to leave in permanently: both datasources are fully - # declarative, and dashboards bind to them by UID, not by database ID. - datasources.settings.deleteDatasources = [ - { name = "Prometheus"; orgId = 1; } - { name = "Loki"; orgId = 1; } - ]; - - datasources.settings.datasources = [ - { - name = "Prometheus"; - type = "prometheus"; - uid = dsUid.prometheus; # fixed, so dashboards can reference it - url = "http://127.0.0.1:${toString ports.prometheus}"; - isDefault = true; - jsonData = { - # Tells Grafana how often the data is actually scraped. This is - # what $__rate_interval is derived from; leaving it unset makes - # Grafana guess 15s and mis-size every rate() window. - timeInterval = scrapeInterval; - httpMethod = "POST"; - }; - } - { - name = "Loki"; - type = "loki"; - uid = dsUid.loki; - url = "http://127.0.0.1:${toString ports.loki}"; - } - ]; - }; - }; - - # Grafana secret key - systemd.services.grafana.serviceConfig.LoadCredential = [ - "secret_key:/run/secrets/grafana-secret_key" - ]; - services.grafana.settings.security.secret_key = - "$__file{/run/credentials/grafana.service/secret_key}"; - - # --------------------------------------------------------------- - # Dashboards - # --------------------------------------------------------------- - environment.etc = lib.mapAttrs' (name: drv: - lib.nameValuePair "grafana-dashboards/${name}.json" { source = drv; } - ) dashboards; - - # --------------------------------------------------------------- - # Reverse proxy - # --------------------------------------------------------------- - services.caddy = { - enable = true; - virtualHosts.":80".extraConfig = '' - reverse_proxy localhost:${toString ports.grafana} - ''; - }; - - system.stateVersion = "26.05"; - }; - }; -} -- cgit v1.3.1