diff options
Diffstat (limited to 'modules')
| -rw-r--r-- | modules/collectors.nix | 368 | ||||
| -rw-r--r-- | modules/common.nix | 46 | ||||
| -rw-r--r-- | modules/dashboards.nix | 75 | ||||
| -rw-r--r-- | modules/default.nix | 13 | ||||
| -rw-r--r-- | modules/packages.nix | 45 | ||||
| -rw-r--r-- | modules/server.nix | 420 |
6 files changed, 967 insertions, 0 deletions
diff --git a/modules/collectors.nix b/modules/collectors.nix new file mode 100644 index 0000000..6f328ee --- /dev/null +++ b/modules/collectors.nix @@ -0,0 +1,368 @@ +# Host-side telemetry sources. +# +# Everything here has to run on the host, not in a container: it reads host hardware and +# kernel state -- /proc, /sys, SMART over raw block devices, ZFS ioctls -- that a +# container has no view of. The half that can be containerised is modules/server.nix. +{ config, lib, pkgs, ... }: + +let + inherit (lib) mkIf mkOption mkEnableOption mkForce types optionals concatMap; + + cfg = config.services.telemetry.collectors; + ports = config.services.telemetry.ports; + + collectorPkgs = pkgs.callPackages ./packages.nix { + # Match the userland to the running kernel module. + zfs = config.boot.zfs.package; + }; + + # services.zfs.autoSnapshot's per-label keep counts, handed to the collector so it can + # report how far each label is from its target rather than just how many snapshots + # exist. Silently empty when auto-snapshotting is off, which the collector handles. + snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ]; + + retentionArgs = optionals config.services.zfs.autoSnapshot.enable ( + concatMap + (label: [ + "--retention" + "${label}=${toString config.services.zfs.autoSnapshot.${label}}" + ]) + snapshotLabels + ); + + # Hardening common to both collectors. Each writes exactly one file into the textfile + # directory and talks to nothing over the network. + hardening = { + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateNetwork = true; + PrivateTmp = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectSystem = "strict"; + RestrictAddressFamilies = [ "AF_UNIX" ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + }; + + mkCollectorService = { description, execStart, extraServiceConfig ? { } }: { + inherit description; + serviceConfig = hardening // { + Type = "oneshot"; + ExecStart = execStart; + User = "root"; + StateDirectory = "prometheus-node-exporter-textfile"; + StateDirectoryMode = "0755"; + # node_exporter runs as a different user and has to be able to read what we write. + UMask = "0022"; + TimeoutStartSec = "180s"; + } // extraServiceConfig; + }; + + mkCollectorTimer = { description, onBootSec, onUnitActiveSec, accuracySec, unit }: { + inherit description; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = onBootSec; + OnUnitActiveSec = onUnitActiveSec; + AccuracySec = accuracySec; + Unit = unit; + }; + }; +in +{ + imports = [ ./common.nix ]; + + options.services.telemetry.collectors = { + enable = mkEnableOption "host-side telemetry collectors (node exporter, SMART, ZFS)"; + + listenAddress = mkOption { + type = types.str; + default = "0.0.0.0"; + description = '' + Address the exporters bind to. The default is deliberately permissive because + the whole point of splitting the halves is that the server scrapes these from + somewhere else -- a container on a bridge, or another machine. Narrow it if the + server runs on this host. + ''; + }; + + openFirewall = mkOption { + type = types.bool; + default = true; + description = '' + Open the node and smartctl exporter ports. Needed whenever the server is not on + this host's loopback, which includes a container with `privateNetwork`. + ''; + }; + + textfileDir = mkOption { + type = types.path; + default = "/var/lib/prometheus-node-exporter-textfile"; + description = '' + Directory the ZFS and SMART collectors write `.prom` files into, and that + node_exporter's textfile collector reads. Changing this also changes the + StateDirectory of both collector units. + ''; + }; + + node = { + enable = mkOption { + type = types.bool; + default = true; + description = "Run the Prometheus node exporter."; + }; + + extraCollectors = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "interrupts" ]; + description = '' + Additional node_exporter collectors, on top of the systemd, processes and + textfile collectors this module enables. + + Note this *adds* to node_exporter's defaults -- cpu, diskstats, filesystem, + meminfo, zfs, hwmon and pressure are already on, so listing those does + nothing. `interrupts` is the notable opt-in: it fills the Node Exporter Full + "IRQ Detail" panel but emits one series per (cpu, irq) pair, which is + thousands of series on a many-core box. + ''; + }; + }; + + smart = { + enable = mkOption { + type = types.bool; + default = true; + description = '' + Run the smartctl exporter and the self-test history collector. + ''; + }; + + devices = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "/dev/sda" "/dev/nvme0" ]; + description = '' + Devices to poll. Empty means auto-discovery via `smartctl --scan`, which is + correct for direct-attached AHCI/HBA disks. Behind a RAID controller you + generally have to list devices explicitly with their `-d` type. + ''; + }; + + maxInterval = mkOption { + type = types.str; + default = "60s"; + description = '' + Floor on how often the exporter actually shells out to smartctl, independent + of how often Prometheus scrapes it. SMART data moves slowly and polling can + spin up idle disks. + ''; + }; + + selfTestInterval = mkOption { + type = types.str; + default = "15min"; + description = '' + How often to re-read the self-test logs. The data only changes when a test + finishes, so this can be far slower than the exporter's own polling. + ''; + }; + }; + + zfs = { + enable = mkOption { + type = types.bool; + default = config.boot.zfs.enabled; + defaultText = lib.literalExpression "config.boot.zfs.enabled"; + description = '' + Collect pool capacity and health, vdev errors, scrub history, dataset + properties and snapshot retention from the zpool/zfs CLIs. + ''; + }; + + interval = mkOption { + type = types.str; + default = "1min"; + description = '' + How often to re-read pool and dataset state. Listing every snapshot on a busy + pool is the slow part; if this starts overlapping its own runs, raise it. + ''; + }; + }; + + logs = { + enable = mkOption { + type = types.bool; + default = true; + description = "Ship the systemd journal to Loki with Grafana Alloy."; + }; + + lokiUrl = mkOption { + type = types.str; + default = "http://localhost:${toString ports.loki}"; + defaultText = lib.literalExpression ''"http://localhost:''${toString config.services.telemetry.ports.loki}"''; + example = "http://metrics.lan:3100"; + description = '' + Base URL of the Loki instance to push to. Point this at wherever + modules/server.nix runs; the `/loki/api/v1/push` path is appended. + ''; + }; + + maxAge = mkOption { + type = types.str; + default = "12h"; + description = '' + How far back into the journal to read on startup. Anything older than this is + skipped rather than replayed into Loki. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + # ------------------------------------------------------------------------------ + # node exporter + # ------------------------------------------------------------------------------ + services.prometheus.exporters.node = mkIf cfg.node.enable { + enable = true; + port = ports.node; + inherit (cfg) listenAddress; + + enabledCollectors = [ + "systemd" # node_systemd_* — the "Systemd" row, and the zfs-* timer panels + "processes" # node_processes_* — "Processes Detailed States", "PIDs", "Threads" + ] ++ cfg.node.extraCollectors; + + extraFlags = [ + "--collector.textfile.directory=${cfg.textfileDir}" + # node_systemd_unit_start_time_seconds, so the dashboards can show when the + # zfs-snapshot-* and zfs-scrub units last ran rather than only whether they + # failed. + "--collector.systemd.enable-start-time-metrics" + ]; + }; + + # Upstream hardens the unit with ProtectHome=true, which makes /home look empty and + # blanks out the filesystem panels for any dataset mounted under it. read-only still + # allows statfs while keeping the sandbox. + systemd.services.prometheus-node-exporter = mkIf cfg.node.enable { + serviceConfig.ProtectHome = mkForce "read-only"; + }; + + # ------------------------------------------------------------------------------ + # SMART + # ------------------------------------------------------------------------------ + services.prometheus.exporters.smartctl = mkIf cfg.smart.enable { + enable = true; + port = ports.smartctl; + inherit (cfg) listenAddress; + inherit (cfg.smart) devices maxInterval; + }; + + systemd.services.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorService { + description = "Collect SMART self-test history and drive identity"; + execStart = "${lib.getExe collectorPkgs.smart-selftest} --output ${cfg.textfileDir}/smart-selftest.prom"; + # Same shape as the smartctl exporter's own hardening: raw device access to talk + # to the disks, and nothing else. + extraServiceConfig = { + AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ]; + CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ]; + DevicePolicy = "closed"; + DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ]; + }; + }); + + systemd.timers.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorTimer { + description = "Collect SMART self-test history periodically"; + onBootSec = "3min"; + onUnitActiveSec = cfg.smart.selfTestInterval; + accuracySec = "1min"; + unit = "smart-selftest-collector.service"; + }); + + # ------------------------------------------------------------------------------ + # ZFS + # ------------------------------------------------------------------------------ + systemd.services.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorService { + description = "Collect ZFS pool, dataset and snapshot metrics"; + execStart = lib.escapeShellArgs ([ + (lib.getExe collectorPkgs.zfs-metrics) + "--output" + "${cfg.textfileDir}/zfs.prom" + ] ++ retentionArgs); + # CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated on it; + # everything else is dropped. + extraServiceConfig = { + CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ]; + SystemCallFilter = [ "@system-service" ]; + }; + }); + + systemd.timers.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorTimer { + description = "Collect ZFS metrics periodically"; + onBootSec = "1min"; + onUnitActiveSec = cfg.zfs.interval; + accuracySec = "5s"; + unit = "zfs-textfile-collector.service"; + }); + + # An earlier version of the ZFS collector wrote zpool.prom. Nothing removes a stale + # textfile once its writer is gone, and node_exporter will happily keep serving + # those frozen metrics forever, so delete it on activation. + systemd.tmpfiles.rules = mkIf cfg.zfs.enable [ + "r ${cfg.textfileDir}/zpool.prom" + ]; + + # ------------------------------------------------------------------------------ + # Journal shipping + # ------------------------------------------------------------------------------ + services.alloy.enable = mkIf cfg.logs.enable true; + + environment.etc."alloy/config.alloy" = mkIf cfg.logs.enable { + text = '' + loki.source.journal "journal" { + forward_to = [loki.write.default.receiver] + relabel_rules = loki.relabel.journal.rules + max_age = "${cfg.logs.maxAge}" + } + + loki.relabel "journal" { + forward_to = [] + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal__hostname"] + target_label = "host" + } + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + } + + loki.write "default" { + endpoint { + url = "${cfg.logs.lokiUrl}/loki/api/v1/push" + } + } + ''; + }; + + # ------------------------------------------------------------------------------ + # Firewall + # ------------------------------------------------------------------------------ + networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall ( + optionals cfg.node.enable [ ports.node ] + ++ optionals cfg.smart.enable [ ports.smartctl ] + ); + }; +} diff --git a/modules/common.nix b/modules/common.nix new file mode 100644 index 0000000..480aa62 --- /dev/null +++ b/modules/common.nix @@ -0,0 +1,46 @@ +# Options both halves need to agree on. +# +# The collectors bind these ports; the server scrapes them and pushes to them. When the +# two halves live in separate NixOS configurations (host + container) there is no shared +# `config` to read, so each side sets its own copy and it is on you to keep them equal — +# which is why they are gathered in one small file rather than scattered. +# +# Imported by both modules/collectors.nix and modules/server.nix. Importing the same +# path twice is harmless: the module system deduplicates imports by path, so a +# configuration that uses both halves does not end up declaring these twice. +{ lib, ... }: + +let + inherit (lib) mkOption types; + + portOption = default: description: mkOption { + inherit default description; + type = types.port; + }; +in +{ + options.services.telemetry.ports = { + node = portOption 9100 '' + Port the Prometheus node exporter listens on. Also where the server's `node` + scrape job points. + ''; + + smartctl = portOption 9633 '' + Port the smartctl exporter listens on. Also where the server's `smartctl` scrape + job points. + ''; + + loki = portOption 3100 '' + Port Loki listens on. Also where the collectors' log shipper pushes to. + ''; + + prometheus = portOption 9090 '' + Port Prometheus listens on. + ''; + + grafana = portOption 3000 '' + Port Grafana listens on. If the reverse proxy is enabled this stays internal and + the proxy is what you connect to. + ''; + }; +} diff --git a/modules/dashboards.nix b/modules/dashboards.nix new file mode 100644 index 0000000..125e3e2 --- /dev/null +++ b/modules/dashboards.nix @@ -0,0 +1,75 @@ +# Grafana dashboards, built into a form file-provisioning can actually use. +# +# Returns an attrset of { name = derivation; }, where each derivation is a single +# dashboard JSON file. modules/server.nix drops these into /etc/grafana-dashboards. +{ lib, runCommand, fetchurl, jq, dsUid }: + +let + # Dashboards exported from grafana.com carry an `__inputs` block and reference their + # datasource as "''${DS_PROMETHEUS}". That placeholder is only substituted by Grafana's + # interactive *import* flow -- file provisioning never runs it, so the uid stays + # literal and every panel fails to resolve a datasource. Substitute at build time. + # + # A dashboard-level `datasource`-type template variable (lowercase, e.g. + # "''${ds_prometheus}") is a different mechanism and resolves fine on its own; this + # only touches names declared in __inputs. + fixupProgram = '' + def dsuid($pluginId): + if $pluginId == "prometheus" then $prom + elif $pluginId == "loki" then $loki + else "" end; + + def subst($name; $uid): + walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end); + + reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId))) + | del(.__inputs, .__requires) + # Provisioned dashboards must not carry an id, or Grafana rejects them on collision + # with whatever already occupies that id in its database. + | .id = null + ''; + + fixup = name: src: runCommand "grafana-dashboard-${name}.json" + { nativeBuildInputs = [ jq ]; } + '' + jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \ + --arg loki ${lib.escapeShellArg dsUid.loki} \ + ${lib.escapeShellArg fixupProgram} < ${src} > "$out" + ''; + + fromGrafanaCom = { name, id, rev, hash }: + fixup name (fetchurl { + url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download"; + inherit hash; + }); +in +{ + # https://grafana.com/grafana/dashboards/1860 + node-exporter-full = fromGrafanaCom { + name = "node-exporter-full"; + id = 1860; + rev = 45; + hash = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY="; + }; + + # https://grafana.com/grafana/dashboards/22604 + smartctl-exporter = fromGrafanaCom { + name = "smartctl-exporter"; + id = 22604; + rev = 2; + hash = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88="; + }; + + # Dashboard 15362 ("ZFS Pool Metrics") used to fill this slot. It is an *InfluxDB* + # dashboard -- its queries are InfluxQL against a `zpool_stats` measurement -- so it + # can never render against Prometheus, however it is provisioned. This is a + # Prometheus-native replacement built on the metrics we actually export: node + # exporter's zfs collector plus the ZFS textfile collector. + # Generated by dashboards/gen_zfs.py. + zfs-pools = fixup "zfs-pools" ../dashboards/zfs-pools.json; + + # CrystalDiskInfo-style per-drive SMART view. Generated by dashboards/gen_smart.py. + # The Good/Caution/Bad verdict and the per-drive row titles both come from the + # recording rules in rules/smartctl.yml, so this one needs those loaded to be useful. + drive-health = fixup "drive-health" ../dashboards/smart.json; +} diff --git a/modules/default.nix b/modules/default.nix new file mode 100644 index 0000000..07aa854 --- /dev/null +++ b/modules/default.nix @@ -0,0 +1,13 @@ +# Both halves at once, for running collectors and the web services on one machine. +# +# Importing this is equivalent to importing collectors.nix and server.nix together. If +# you are splitting them -- collectors on the host, server in a container -- import the +# two modules separately instead; each is self-contained. +{ ... }: + +{ + imports = [ + ./collectors.nix + ./server.nix + ]; +} diff --git a/modules/packages.nix b/modules/packages.nix new file mode 100644 index 0000000..5e76f10 --- /dev/null +++ b/modules/packages.nix @@ -0,0 +1,45 @@ +# The two textfile collectors, as ordinary packages. +# +# They are wrappers rather than inline `writeShellApplication` bodies in the module so +# that the Python is runnable by hand -- `zfs-metrics --output /tmp/x.prom` behaves +# exactly as the timer does, which is how every one of these metrics was validated. +# Neither wrapper hardcodes an output path; the unit passes it. +{ lib, writeShellApplication, python3, smartmontools, zfs }: + +{ + # Pool capacity and health, vdev errors, scrub history, dataset properties and + # snapshot retention -- none of which node_exporter's built-in zfs collector reports, + # because it reads /proc/spl/kstat/zfs and that only carries ARC statistics and objset + # counters. OpenZFS 2.2+ also dropped the per-pool `io` kstat, so node_zfs_zpool_* is + # largely empty. This shells out to the zpool/zfs CLIs instead. + # + # runtimeInputs is the whole reason for the wrapper: the collector must run against + # the same zfs userland as the running kernel module. + zfs-metrics = writeShellApplication { + name = "zfs-metrics"; + runtimeInputs = [ zfs ]; + text = '' + exec ${python3.interpreter} ${../collectors/zfs-metrics.py} "$@" + ''; + meta = { + description = "Export ZFS pool, dataset and snapshot metrics in Prometheus format"; + mainProgram = "zfs-metrics"; + }; + }; + + # smartctl_exporter reads only the entry *count* out of each drive's self-test log and + # discards the log table, so it cannot say when a drive last ran a short or extended + # test or whether it passed. It also has no way to render a drive's identity as a + # display string, which the dashboards need for row titles. Both come from here. + smart-selftest = writeShellApplication { + name = "smart-selftest"; + runtimeInputs = [ smartmontools ]; + text = '' + exec ${python3.interpreter} ${../collectors/smart-selftest.py} "$@" + ''; + meta = { + description = "Export SMART self-test history and drive identity in Prometheus format"; + mainProgram = "smart-selftest"; + }; + }; +} diff --git a/modules/server.nix b/modules/server.nix new file mode 100644 index 0000000..1996f5e --- /dev/null +++ b/modules/server.nix @@ -0,0 +1,420 @@ +# The web half: Prometheus, Loki, Grafana and an optional reverse proxy. +# +# Nothing here reads host hardware, so this is the part you can put in a container. It +# reaches the collectors over the network -- see `collectorHosts`. +{ config, lib, pkgs, ... }: + +let + inherit (lib) mkIf mkOption mkEnableOption mkDefault types optionals filterAttrs mapAttrs' nameValuePair; + + cfg = config.services.telemetry.server; + ports = config.services.telemetry.ports; + + # Fixed, because the generated dashboards reference them literally. dashboards/panels.py + # emits {"type": "prometheus", "uid": "prometheus"} on every panel, and the jq fixup in + # dashboards.nix rewrites grafana.com's placeholders to the same values. Changing these + # would silently break every panel, so they are not options. + dsUid = { prometheus = "prometheus"; loki = "loki"; }; + + # A `host` label carrying the machine name with the port stripped off. + # + # `instance` cannot do this job. The same machine is scraped twice on different ports + # -- the node exporter on :9100 serves the textfile metrics (zfs_*, smart_selftest_*, + # smart_drive_label) while smartctl_* comes from its own exporter on :9633 -- so one + # machine has two `instance` values, and the recording rules join straight across that + # boundary. `host` is equal on both, which is what makes the dashboards' host picker + # and those joins work at the same time. + # + # Note the regex is fully anchored by Prometheus, and only handles host:port. A + # bracketed IPv6 literal needs an explicit label via `extraScrapeConfigs` instead. + hostLabelRelabel = [{ + source_labels = [ "__address__" ]; + regex = "([^:]+)(?::[0-9]+)?"; + target_label = "host"; + replacement = "$1"; + }]; + + builtinDashboards = pkgs.callPackage ./dashboards.nix { inherit dsUid; }; + + selected = filterAttrs (name: _: cfg.grafana.dashboards.${name} or false) builtinDashboards; + + dashboards = selected // cfg.grafana.extraDashboards; + + dashboardOption = description: mkOption { + inherit description; + type = types.bool; + default = true; + }; +in +{ + imports = [ ./common.nix ]; + + options.services.telemetry.server = { + enable = mkEnableOption "the Prometheus/Loki/Grafana server"; + + collectorHosts = mkOption { + type = types.listOf types.str; + default = [ "localhost" ]; + example = [ "poweredge" "optiplex" ]; + description = '' + Machines running modules/collectors.nix. Each is scraped on both the node and + smartctl ports from {option}`services.telemetry.ports`, so this is normally the + only thing on the server side that has to know about the hosts. + + Each entry also becomes the value of the `host` label on everything scraped + from that machine, which is what the dashboards' host picker selects on. Use + the same spelling you would type in a browser; if you list an IP address, that + is what the picker will show. + + Set {option}`services.telemetry.server.prometheus.nodeTargets` or + {option}`services.telemetry.server.prometheus.smartctlTargets` to take over + target generation entirely. + ''; + }; + + listenAddress = mkOption { + type = types.str; + default = "0.0.0.0"; + description = '' + Address Loki, Prometheus and Grafana bind to. Loki has to be reachable from + every host running the collectors, so this defaults to all interfaces. + ''; + }; + + openFirewall = mkOption { + type = types.bool; + default = true; + description = '' + Open the ports of whichever services are enabled: Loki (the collectors push to + it), Prometheus (useful for checking scrape health directly), and either the + proxy or Grafana. + ''; + }; + + prometheus = { + enable = mkOption { + type = types.bool; + default = true; + description = "Run Prometheus."; + }; + + scrapeInterval = mkOption { + type = types.str; + default = "15s"; + description = '' + Global scrape interval, and the value handed to Grafana's Prometheus + datasource as `timeInterval`. + + These two MUST agree. Grafana derives `$__rate_interval` from `timeInterval`, + and if it guesses wrong then `rate()` windows are mis-sized and every + rate-based panel intermittently renders "No data" -- which is exactly what + Prometheus' own 1m default does to the Node Exporter Full dashboard. Setting + it here sets both. + ''; + }; + + nodeTargets = mkOption { + type = types.listOf types.str; + default = map (host: "${host}:${toString ports.node}") cfg.collectorHosts; + defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the node exporter port"; + description = "Targets for the `node` scrape job."; + }; + + smartctlTargets = mkOption { + type = types.listOf types.str; + default = map (host: "${host}:${toString ports.smartctl}") cfg.collectorHosts; + defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the smartctl exporter port"; + description = "Targets for the `smartctl` scrape job. Empty disables the job."; + }; + + smartctlScrapeInterval = mkOption { + type = types.str; + default = "60s"; + description = '' + Scrape interval for the smartctl job, which opts out of the global one: + querying SMART is slow and the data barely moves. Safe because the drive + dashboards use no `rate()` queries at all. + ''; + }; + + ruleFiles = mkOption { + type = types.listOf types.path; + default = [ ../rules/smartctl.yml ]; + defaultText = lib.literalExpression "[ ../rules/smartctl.yml ]"; + description = '' + Recording rule files. The default computes the per-drive Good/Caution/Bad + verdict and the row-title labels that the drive health dashboard displays, so + drop it only if you also drop that dashboard. + + {option}`services.prometheus.checkConfig` runs promtool over these at build + time, so a broken rule fails the build rather than the service. + ''; + }; + + extraScrapeConfigs = mkOption { + type = types.listOf types.attrs; + default = [ ]; + description = "Additional scrape jobs, appended to the generated ones."; + }; + + retentionTime = mkOption { + type = types.str; + default = "90d"; + description = "How long to keep samples."; + }; + }; + + loki = { + enable = mkOption { + type = types.bool; + default = true; + description = "Run Loki to receive journal logs from the collectors."; + }; + + dataDir = mkOption { + type = types.path; + default = "/var/lib/loki"; + description = "Where Loki stores its chunks and index."; + }; + }; + + grafana = { + enable = mkOption { + type = types.bool; + default = true; + description = "Run Grafana, provisioned with datasources and dashboards."; + }; + + secretKeyFile = mkOption { + type = types.nullOr types.path; + default = null; + example = lib.literalExpression "config.sops.secrets.grafana-secret-key.path"; + description = '' + File holding Grafana's `security.secret_key`. Loaded through systemd's + `LoadCredential`, so the file only has to be readable by root at start time + and never lands in the Nix store. + + Deliberately a plain path rather than a sops-nix integration: wire it up to + whatever secret manager you already use. + ''; + }; + + dashboards = { + node-exporter-full = dashboardOption "Provision grafana.com dashboard 1860 (Node Exporter Full)."; + smartctl-exporter = dashboardOption "Provision grafana.com dashboard 22604 (Smartctl Exporter)."; + zfs-pools = dashboardOption "Provision the ZFS pools dashboard. Needs the ZFS collector."; + drive-health = dashboardOption "Provision the CrystalDiskInfo-style drive health dashboard. Needs the SMART collector and the smartctl recording rules."; + }; + + extraDashboards = mkOption { + type = types.attrsOf types.path; + default = { }; + example = lib.literalExpression ''{ my-dashboard = ./my-dashboard.json; }''; + description = '' + Extra dashboards to provision, as { name = path-to-json; }. These are passed + through unmodified, so they must already reference the datasource UIDs + "prometheus" and "loki". + ''; + }; + }; + + proxy = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Run Caddy in front of Grafana. Off by default because a reverse proxy is + usually something you already have; turn it on to get Grafana on port 80 + without exposing its own port. + ''; + }; + + virtualHost = mkOption { + type = types.str; + default = ":80"; + example = "metrics.example.com"; + description = '' + Caddy virtual host. A bare hostname makes Caddy provision a certificate + automatically; ":80" serves plain HTTP on every name. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + # Grafana uses this key to encrypt datasource credentials in its database, and + # nixpkgs stopped shipping a default in 26.05. Caught here rather than left to + # Grafana's own assertion, which names an option this module manages for you. + assertions = [{ + assertion = !cfg.grafana.enable || cfg.grafana.secretKeyFile != null; + message = '' + services.telemetry.server.grafana.secretKeyFile must be set when Grafana is + enabled. Generate a key (`openssl rand -base64 32`), store it with whatever + secret manager you use, and point this option at the resulting file. + ''; + }]; + + # -------------------------------------------------------------------------------- + # Loki + # -------------------------------------------------------------------------------- + services.loki = mkIf cfg.loki.enable { + enable = true; + configuration = { + auth_enabled = false; + + server = { + http_listen_port = ports.loki; + # Explicit: the collectors have to reach this from another host or container. + http_listen_address = cfg.listenAddress; + }; + + common = { + path_prefix = cfg.loki.dataDir; + storage.filesystem = { + chunks_directory = "${cfg.loki.dataDir}/chunks"; + rules_directory = "${cfg.loki.dataDir}/rules"; + }; + ring = { + instance_addr = "127.0.0.1"; + kvstore.store = "inmemory"; + }; + replication_factor = 1; + }; + + schema_config.configs = [{ + from = "2024-01-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { prefix = "index_"; period = "24h"; }; + }]; + }; + }; + + # -------------------------------------------------------------------------------- + # Prometheus + # -------------------------------------------------------------------------------- + services.prometheus = mkIf cfg.prometheus.enable { + enable = true; + port = ports.prometheus; + listenAddress = cfg.listenAddress; + retentionTime = cfg.prometheus.retentionTime; + + globalConfig = { + scrape_interval = cfg.prometheus.scrapeInterval; + evaluation_interval = cfg.prometheus.scrapeInterval; + scrape_timeout = "10s"; + }; + + inherit (cfg.prometheus) ruleFiles; + + scrapeConfigs = + optionals (cfg.prometheus.nodeTargets != [ ]) [{ + job_name = "node"; + relabel_configs = hostLabelRelabel; + static_configs = [{ targets = cfg.prometheus.nodeTargets; }]; + }] + ++ optionals (cfg.prometheus.smartctlTargets != [ ]) [{ + job_name = "smartctl"; + scrape_interval = cfg.prometheus.smartctlScrapeInterval; + scrape_timeout = "30s"; + relabel_configs = hostLabelRelabel; + static_configs = [{ targets = cfg.prometheus.smartctlTargets; }]; + }] + ++ cfg.prometheus.extraScrapeConfigs; + }; + + # -------------------------------------------------------------------------------- + # Grafana + # -------------------------------------------------------------------------------- + services.grafana = mkIf cfg.grafana.enable { + enable = true; + + settings = { + server = { + http_addr = mkDefault cfg.listenAddress; + http_port = mkDefault ports.grafana; + }; + } + # A file provider rather than a literal, so the key is never written into a + # world-readable store path. The credential is mounted by LoadCredential below. + // lib.optionalAttrs (cfg.grafana.secretKeyFile != null) { + security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}"; + }; + + provision = { + enable = true; + + dashboards.settings.providers = [{ + name = "provisioned"; + disableDeletion = true; + options = { + path = "/etc/grafana-dashboards"; + foldersFromFilesStructure = true; + }; + }]; + + # Grafana cannot change the UID of an existing datasource in place. Its + # provisioner finds the datasource by name, then calls UpdateDataSource, which + # re-fetches using `id AND uid` -- with a new uid that row does not match, so it + # fails with "data source not found" and Grafana refuses to start. Deleting + # first turns the upsert into a plain insert; the provisioner runs deletes + # before inserts for exactly this case and suppresses the deletion event because + # the datasource is about to be re-created. Safe permanently: both datasources + # are fully declarative and dashboards bind to them by UID, not database ID. + datasources.settings.deleteDatasources = + [{ name = "Prometheus"; orgId = 1; }] + ++ optionals cfg.loki.enable [{ name = "Loki"; orgId = 1; }]; + + datasources.settings.datasources = + optionals cfg.prometheus.enable [{ + name = "Prometheus"; + type = "prometheus"; + uid = dsUid.prometheus; + url = "http://127.0.0.1:${toString ports.prometheus}"; + isDefault = true; + jsonData = { + # What $__rate_interval is derived from. Leaving it unset makes Grafana + # assume 15s and mis-size every rate() window. See scrapeInterval above. + timeInterval = cfg.prometheus.scrapeInterval; + httpMethod = "POST"; + }; + }] + ++ optionals cfg.loki.enable [{ + name = "Loki"; + type = "loki"; + uid = dsUid.loki; + url = "http://127.0.0.1:${toString ports.loki}"; + }]; + }; + }; + + systemd.services.grafana = mkIf (cfg.grafana.enable && cfg.grafana.secretKeyFile != null) { + serviceConfig.LoadCredential = [ "secret_key:${cfg.grafana.secretKeyFile}" ]; + }; + + environment.etc = mkIf cfg.grafana.enable ( + mapAttrs' (name: src: nameValuePair "grafana-dashboards/${name}.json" { source = src; }) + dashboards + ); + + # -------------------------------------------------------------------------------- + # Reverse proxy + # -------------------------------------------------------------------------------- + services.caddy = mkIf cfg.proxy.enable { + enable = true; + virtualHosts.${cfg.proxy.virtualHost}.extraConfig = '' + reverse_proxy localhost:${toString ports.grafana} + ''; + }; + + # -------------------------------------------------------------------------------- + # Firewall + # -------------------------------------------------------------------------------- + networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall ( + optionals cfg.loki.enable [ ports.loki ] + ++ optionals cfg.prometheus.enable [ ports.prometheus ] + ++ (if cfg.proxy.enable then [ 80 443 ] else optionals cfg.grafana.enable [ ports.grafana ]) + ); + }; +} |