diff options
Diffstat (limited to 'modules/server.nix')
| -rw-r--r-- | modules/server.nix | 420 |
1 files changed, 420 insertions, 0 deletions
diff --git a/modules/server.nix b/modules/server.nix new file mode 100644 index 0000000..1996f5e --- /dev/null +++ b/modules/server.nix @@ -0,0 +1,420 @@ +# The web half: Prometheus, Loki, Grafana and an optional reverse proxy. +# +# Nothing here reads host hardware, so this is the part you can put in a container. It +# reaches the collectors over the network -- see `collectorHosts`. +{ config, lib, pkgs, ... }: + +let + inherit (lib) mkIf mkOption mkEnableOption mkDefault types optionals filterAttrs mapAttrs' nameValuePair; + + cfg = config.services.telemetry.server; + ports = config.services.telemetry.ports; + + # Fixed, because the generated dashboards reference them literally. dashboards/panels.py + # emits {"type": "prometheus", "uid": "prometheus"} on every panel, and the jq fixup in + # dashboards.nix rewrites grafana.com's placeholders to the same values. Changing these + # would silently break every panel, so they are not options. + dsUid = { prometheus = "prometheus"; loki = "loki"; }; + + # A `host` label carrying the machine name with the port stripped off. + # + # `instance` cannot do this job. The same machine is scraped twice on different ports + # -- the node exporter on :9100 serves the textfile metrics (zfs_*, smart_selftest_*, + # smart_drive_label) while smartctl_* comes from its own exporter on :9633 -- so one + # machine has two `instance` values, and the recording rules join straight across that + # boundary. `host` is equal on both, which is what makes the dashboards' host picker + # and those joins work at the same time. + # + # Note the regex is fully anchored by Prometheus, and only handles host:port. A + # bracketed IPv6 literal needs an explicit label via `extraScrapeConfigs` instead. + hostLabelRelabel = [{ + source_labels = [ "__address__" ]; + regex = "([^:]+)(?::[0-9]+)?"; + target_label = "host"; + replacement = "$1"; + }]; + + builtinDashboards = pkgs.callPackage ./dashboards.nix { inherit dsUid; }; + + selected = filterAttrs (name: _: cfg.grafana.dashboards.${name} or false) builtinDashboards; + + dashboards = selected // cfg.grafana.extraDashboards; + + dashboardOption = description: mkOption { + inherit description; + type = types.bool; + default = true; + }; +in +{ + imports = [ ./common.nix ]; + + options.services.telemetry.server = { + enable = mkEnableOption "the Prometheus/Loki/Grafana server"; + + collectorHosts = mkOption { + type = types.listOf types.str; + default = [ "localhost" ]; + example = [ "poweredge" "optiplex" ]; + description = '' + Machines running modules/collectors.nix. Each is scraped on both the node and + smartctl ports from {option}`services.telemetry.ports`, so this is normally the + only thing on the server side that has to know about the hosts. + + Each entry also becomes the value of the `host` label on everything scraped + from that machine, which is what the dashboards' host picker selects on. Use + the same spelling you would type in a browser; if you list an IP address, that + is what the picker will show. + + Set {option}`services.telemetry.server.prometheus.nodeTargets` or + {option}`services.telemetry.server.prometheus.smartctlTargets` to take over + target generation entirely. + ''; + }; + + listenAddress = mkOption { + type = types.str; + default = "0.0.0.0"; + description = '' + Address Loki, Prometheus and Grafana bind to. Loki has to be reachable from + every host running the collectors, so this defaults to all interfaces. + ''; + }; + + openFirewall = mkOption { + type = types.bool; + default = true; + description = '' + Open the ports of whichever services are enabled: Loki (the collectors push to + it), Prometheus (useful for checking scrape health directly), and either the + proxy or Grafana. + ''; + }; + + prometheus = { + enable = mkOption { + type = types.bool; + default = true; + description = "Run Prometheus."; + }; + + scrapeInterval = mkOption { + type = types.str; + default = "15s"; + description = '' + Global scrape interval, and the value handed to Grafana's Prometheus + datasource as `timeInterval`. + + These two MUST agree. Grafana derives `$__rate_interval` from `timeInterval`, + and if it guesses wrong then `rate()` windows are mis-sized and every + rate-based panel intermittently renders "No data" -- which is exactly what + Prometheus' own 1m default does to the Node Exporter Full dashboard. Setting + it here sets both. + ''; + }; + + nodeTargets = mkOption { + type = types.listOf types.str; + default = map (host: "${host}:${toString ports.node}") cfg.collectorHosts; + defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the node exporter port"; + description = "Targets for the `node` scrape job."; + }; + + smartctlTargets = mkOption { + type = types.listOf types.str; + default = map (host: "${host}:${toString ports.smartctl}") cfg.collectorHosts; + defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the smartctl exporter port"; + description = "Targets for the `smartctl` scrape job. Empty disables the job."; + }; + + smartctlScrapeInterval = mkOption { + type = types.str; + default = "60s"; + description = '' + Scrape interval for the smartctl job, which opts out of the global one: + querying SMART is slow and the data barely moves. Safe because the drive + dashboards use no `rate()` queries at all. + ''; + }; + + ruleFiles = mkOption { + type = types.listOf types.path; + default = [ ../rules/smartctl.yml ]; + defaultText = lib.literalExpression "[ ../rules/smartctl.yml ]"; + description = '' + Recording rule files. The default computes the per-drive Good/Caution/Bad + verdict and the row-title labels that the drive health dashboard displays, so + drop it only if you also drop that dashboard. + + {option}`services.prometheus.checkConfig` runs promtool over these at build + time, so a broken rule fails the build rather than the service. + ''; + }; + + extraScrapeConfigs = mkOption { + type = types.listOf types.attrs; + default = [ ]; + description = "Additional scrape jobs, appended to the generated ones."; + }; + + retentionTime = mkOption { + type = types.str; + default = "90d"; + description = "How long to keep samples."; + }; + }; + + loki = { + enable = mkOption { + type = types.bool; + default = true; + description = "Run Loki to receive journal logs from the collectors."; + }; + + dataDir = mkOption { + type = types.path; + default = "/var/lib/loki"; + description = "Where Loki stores its chunks and index."; + }; + }; + + grafana = { + enable = mkOption { + type = types.bool; + default = true; + description = "Run Grafana, provisioned with datasources and dashboards."; + }; + + secretKeyFile = mkOption { + type = types.nullOr types.path; + default = null; + example = lib.literalExpression "config.sops.secrets.grafana-secret-key.path"; + description = '' + File holding Grafana's `security.secret_key`. Loaded through systemd's + `LoadCredential`, so the file only has to be readable by root at start time + and never lands in the Nix store. + + Deliberately a plain path rather than a sops-nix integration: wire it up to + whatever secret manager you already use. + ''; + }; + + dashboards = { + node-exporter-full = dashboardOption "Provision grafana.com dashboard 1860 (Node Exporter Full)."; + smartctl-exporter = dashboardOption "Provision grafana.com dashboard 22604 (Smartctl Exporter)."; + zfs-pools = dashboardOption "Provision the ZFS pools dashboard. Needs the ZFS collector."; + drive-health = dashboardOption "Provision the CrystalDiskInfo-style drive health dashboard. Needs the SMART collector and the smartctl recording rules."; + }; + + extraDashboards = mkOption { + type = types.attrsOf types.path; + default = { }; + example = lib.literalExpression ''{ my-dashboard = ./my-dashboard.json; }''; + description = '' + Extra dashboards to provision, as { name = path-to-json; }. These are passed + through unmodified, so they must already reference the datasource UIDs + "prometheus" and "loki". + ''; + }; + }; + + proxy = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Run Caddy in front of Grafana. Off by default because a reverse proxy is + usually something you already have; turn it on to get Grafana on port 80 + without exposing its own port. + ''; + }; + + virtualHost = mkOption { + type = types.str; + default = ":80"; + example = "metrics.example.com"; + description = '' + Caddy virtual host. A bare hostname makes Caddy provision a certificate + automatically; ":80" serves plain HTTP on every name. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + # Grafana uses this key to encrypt datasource credentials in its database, and + # nixpkgs stopped shipping a default in 26.05. Caught here rather than left to + # Grafana's own assertion, which names an option this module manages for you. + assertions = [{ + assertion = !cfg.grafana.enable || cfg.grafana.secretKeyFile != null; + message = '' + services.telemetry.server.grafana.secretKeyFile must be set when Grafana is + enabled. Generate a key (`openssl rand -base64 32`), store it with whatever + secret manager you use, and point this option at the resulting file. + ''; + }]; + + # -------------------------------------------------------------------------------- + # Loki + # -------------------------------------------------------------------------------- + services.loki = mkIf cfg.loki.enable { + enable = true; + configuration = { + auth_enabled = false; + + server = { + http_listen_port = ports.loki; + # Explicit: the collectors have to reach this from another host or container. + http_listen_address = cfg.listenAddress; + }; + + common = { + path_prefix = cfg.loki.dataDir; + storage.filesystem = { + chunks_directory = "${cfg.loki.dataDir}/chunks"; + rules_directory = "${cfg.loki.dataDir}/rules"; + }; + ring = { + instance_addr = "127.0.0.1"; + kvstore.store = "inmemory"; + }; + replication_factor = 1; + }; + + schema_config.configs = [{ + from = "2024-01-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { prefix = "index_"; period = "24h"; }; + }]; + }; + }; + + # -------------------------------------------------------------------------------- + # Prometheus + # -------------------------------------------------------------------------------- + services.prometheus = mkIf cfg.prometheus.enable { + enable = true; + port = ports.prometheus; + listenAddress = cfg.listenAddress; + retentionTime = cfg.prometheus.retentionTime; + + globalConfig = { + scrape_interval = cfg.prometheus.scrapeInterval; + evaluation_interval = cfg.prometheus.scrapeInterval; + scrape_timeout = "10s"; + }; + + inherit (cfg.prometheus) ruleFiles; + + scrapeConfigs = + optionals (cfg.prometheus.nodeTargets != [ ]) [{ + job_name = "node"; + relabel_configs = hostLabelRelabel; + static_configs = [{ targets = cfg.prometheus.nodeTargets; }]; + }] + ++ optionals (cfg.prometheus.smartctlTargets != [ ]) [{ + job_name = "smartctl"; + scrape_interval = cfg.prometheus.smartctlScrapeInterval; + scrape_timeout = "30s"; + relabel_configs = hostLabelRelabel; + static_configs = [{ targets = cfg.prometheus.smartctlTargets; }]; + }] + ++ cfg.prometheus.extraScrapeConfigs; + }; + + # -------------------------------------------------------------------------------- + # Grafana + # -------------------------------------------------------------------------------- + services.grafana = mkIf cfg.grafana.enable { + enable = true; + + settings = { + server = { + http_addr = mkDefault cfg.listenAddress; + http_port = mkDefault ports.grafana; + }; + } + # A file provider rather than a literal, so the key is never written into a + # world-readable store path. The credential is mounted by LoadCredential below. + // lib.optionalAttrs (cfg.grafana.secretKeyFile != null) { + security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}"; + }; + + provision = { + enable = true; + + dashboards.settings.providers = [{ + name = "provisioned"; + disableDeletion = true; + options = { + path = "/etc/grafana-dashboards"; + foldersFromFilesStructure = true; + }; + }]; + + # Grafana cannot change the UID of an existing datasource in place. Its + # provisioner finds the datasource by name, then calls UpdateDataSource, which + # re-fetches using `id AND uid` -- with a new uid that row does not match, so it + # fails with "data source not found" and Grafana refuses to start. Deleting + # first turns the upsert into a plain insert; the provisioner runs deletes + # before inserts for exactly this case and suppresses the deletion event because + # the datasource is about to be re-created. Safe permanently: both datasources + # are fully declarative and dashboards bind to them by UID, not database ID. + datasources.settings.deleteDatasources = + [{ name = "Prometheus"; orgId = 1; }] + ++ optionals cfg.loki.enable [{ name = "Loki"; orgId = 1; }]; + + datasources.settings.datasources = + optionals cfg.prometheus.enable [{ + name = "Prometheus"; + type = "prometheus"; + uid = dsUid.prometheus; + url = "http://127.0.0.1:${toString ports.prometheus}"; + isDefault = true; + jsonData = { + # What $__rate_interval is derived from. Leaving it unset makes Grafana + # assume 15s and mis-size every rate() window. See scrapeInterval above. + timeInterval = cfg.prometheus.scrapeInterval; + httpMethod = "POST"; + }; + }] + ++ optionals cfg.loki.enable [{ + name = "Loki"; + type = "loki"; + uid = dsUid.loki; + url = "http://127.0.0.1:${toString ports.loki}"; + }]; + }; + }; + + systemd.services.grafana = mkIf (cfg.grafana.enable && cfg.grafana.secretKeyFile != null) { + serviceConfig.LoadCredential = [ "secret_key:${cfg.grafana.secretKeyFile}" ]; + }; + + environment.etc = mkIf cfg.grafana.enable ( + mapAttrs' (name: src: nameValuePair "grafana-dashboards/${name}.json" { source = src; }) + dashboards + ); + + # -------------------------------------------------------------------------------- + # Reverse proxy + # -------------------------------------------------------------------------------- + services.caddy = mkIf cfg.proxy.enable { + enable = true; + virtualHosts.${cfg.proxy.virtualHost}.extraConfig = '' + reverse_proxy localhost:${toString ports.grafana} + ''; + }; + + # -------------------------------------------------------------------------------- + # Firewall + # -------------------------------------------------------------------------------- + networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall ( + optionals cfg.loki.enable [ ports.loki ] + ++ optionals cfg.prometheus.enable [ ports.prometheus ] + ++ (if cfg.proxy.enable then [ 80 443 ] else optionals cfg.grafana.enable [ ports.grafana ]) + ); + }; +} |