path: root/modules/server.nix

# The web half: Prometheus, Loki, Grafana and an optional reverse proxy.
#
# Nothing here reads host hardware, so this is the part you can put in a container. It
# reaches the collectors over the network -- see `collectorHosts`.
{ config, lib, pkgs, ... }:

let
  inherit (lib) mkIf mkOption mkEnableOption mkDefault types optionals filterAttrs mapAttrs' nameValuePair;

  cfg = config.services.telemetry.server;
  ports = config.services.telemetry.ports;

  # Fixed, because the generated dashboards reference them literally. dashboards/panels.py
  # emits {"type": "prometheus", "uid": "prometheus"} on every panel, and the jq fixup in
  # dashboards.nix rewrites grafana.com's placeholders to the same values. Changing these
  # would silently break every panel, so they are not options.
  dsUid = { prometheus = "prometheus"; loki = "loki"; };

  # A `host` label carrying the machine name with the port stripped off.
  #
  # `instance` cannot do this job. The same machine is scraped twice on different ports
  # -- the node exporter on :9100 serves the textfile metrics (zfs_*, smart_selftest_*,
  # smart_drive_label) while smartctl_* comes from its own exporter on :9633 -- so one
  # machine has two `instance` values, and the recording rules join straight across that
  # boundary. `host` is equal on both, which is what makes the dashboards' host picker
  # and those joins work at the same time.
  #
  # Note the regex is fully anchored by Prometheus, and only handles host:port. A
  # bracketed IPv6 literal needs an explicit label via `extraScrapeConfigs` instead.
  hostLabelRelabel = [{
    source_labels = [ "__address__" ];
    regex = "([^:]+)(?::[0-9]+)?";
    target_label = "host";
    replacement = "$1";
  }];

  # Where Grafana's secret key actually comes from: a file you supply, or one this
  # module generates into Grafana's data directory on first start.
  secretKeyPath =
    if cfg.grafana.generateSecretKey
    then "${config.services.grafana.dataDir}/secret-key"
    else cfg.grafana.secretKeyFile;

  builtinDashboards = pkgs.callPackage ./dashboards.nix { inherit dsUid; };

  selected = filterAttrs (name: _: cfg.grafana.dashboards.${name} or false) builtinDashboards;

  dashboards = selected // cfg.grafana.extraDashboards;

  dashboardOption = description: mkOption {
    inherit description;
    type = types.bool;
    default = true;
  };
in
{
  imports = [ ./common.nix ];

  options.services.telemetry.server = {
    enable = mkEnableOption "the Prometheus/Loki/Grafana server";

    collectorHosts = mkOption {
      type = types.listOf types.str;
      default = [ "localhost" ];
      example = [ "poweredge" "optiplex" ];
      description = ''
        Machines running modules/collectors.nix. Each is scraped on both the node and
        smartctl ports from {option}`services.telemetry.ports`, so this is normally the
        only thing on the server side that has to know about the hosts.

        Each entry also becomes the value of the `host` label on everything scraped
        from that machine, which is what the dashboards' host picker selects on. Use
        the same spelling you would type in a browser; if you list an IP address, that
        is what the picker will show.

        Set {option}`services.telemetry.server.prometheus.nodeTargets` or
        {option}`services.telemetry.server.prometheus.smartctlTargets` to take over
        target generation entirely.
      '';
    };

    listenAddress = mkOption {
      type = types.str;
      default = "0.0.0.0";
      description = ''
        Address Loki, Prometheus and Grafana bind to. Loki has to be reachable from
        every host running the collectors, so this defaults to all interfaces.
      '';
    };

    openFirewall = mkOption {
      type = types.bool;
      default = true;
      description = ''
        Open the ports of whichever services are enabled: Loki (the collectors push to
        it), Prometheus (useful for checking scrape health directly), and either the
        proxy or Grafana.
      '';
    };

    prometheus = {
      enable = mkOption {
        type = types.bool;
        default = true;
        description = "Run Prometheus.";
      };

      scrapeInterval = mkOption {
        type = types.str;
        default = "15s";
        description = ''
          Global scrape interval, and the value handed to Grafana's Prometheus
          datasource as `timeInterval`.

          These two MUST agree. Grafana derives `$__rate_interval` from `timeInterval`,
          and if it guesses wrong then `rate()` windows are mis-sized and every
          rate-based panel intermittently renders "No data" -- which is exactly what
          Prometheus' own 1m default does to the Node Exporter Full dashboard. Setting
          it here sets both.
        '';
      };

      nodeTargets = mkOption {
        type = types.listOf types.str;
        default = map (host: "${host}:${toString ports.node}") cfg.collectorHosts;
        defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the node exporter port";
        description = "Targets for the `node` scrape job.";
      };

      smartctlTargets = mkOption {
        type = types.listOf types.str;
        default = map (host: "${host}:${toString ports.smartctl}") cfg.collectorHosts;
        defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the smartctl exporter port";
        description = "Targets for the `smartctl` scrape job. Empty disables the job.";
      };

      smartctlScrapeInterval = mkOption {
        type = types.str;
        default = "60s";
        description = ''
          Scrape interval for the smartctl job, which opts out of the global one:
          querying SMART is slow and the data barely moves. Safe because the drive
          dashboards use no `rate()` queries at all.
        '';
      };

      ruleFiles = mkOption {
        type = types.listOf types.path;
        default = [ ../rules/smartctl.yml ];
        defaultText = lib.literalExpression "[ ../rules/smartctl.yml ]";
        description = ''
          Recording rule files. The default computes the per-drive Good/Caution/Bad
          verdict and the row-title labels that the drive health dashboard displays, so
          drop it only if you also drop that dashboard.

          {option}`services.prometheus.checkConfig` runs promtool over these at build
          time, so a broken rule fails the build rather than the service.
        '';
      };

      extraScrapeConfigs = mkOption {
        type = types.listOf types.attrs;
        default = [ ];
        description = "Additional scrape jobs, appended to the generated ones.";
      };

      retentionTime = mkOption {
        type = types.str;
        default = "90d";
        description = "How long to keep samples.";
      };
    };

    loki = {
      enable = mkOption {
        type = types.bool;
        default = true;
        description = "Run Loki to receive journal logs from the collectors.";
      };

      dataDir = mkOption {
        type = types.path;
        default = "/var/lib/loki";
        description = "Where Loki stores its chunks and index.";
      };
    };

    grafana = {
      enable = mkOption {
        type = types.bool;
        default = true;
        description = "Run Grafana, provisioned with datasources and dashboards.";
      };

      generateSecretKey = mkOption {
        type = types.bool;
        default = cfg.grafana.secretKeyFile == null;
        defaultText = lib.literalMD "`true` unless {option}`secretKeyFile` is set";
        description = ''
          Generate a random `security.secret_key` on first start and keep it in
          Grafana's data directory, instead of supplying one yourself.

          Grafana needs this key to be *stable* -- it encrypts credentials stored in
          Grafana's database, and they become undecryptable if it changes. It has an
          upstream default, but that default is a constant compiled into every Grafana
          release, so leaving it unset means encrypting with a publicly known key;
          nixpkgs asserts rather than let that happen silently.

          The generated file is written by the grafana user with mode 0600, so only
          Grafana and root can read it. Note it then lives beside the database it
          protects: anyone who can read `grafana.db` can read the key too. That is a
          real limitation, and it is why {option}`secretKeyFile` exists -- but the key
          is still per-install and secret from anyone without filesystem access, which
          is the property that actually matters here. Prefer `secretKeyFile` once
          Grafana's database holds credentials worth protecting, such as a datasource
          password or a contact-point API token.
        '';
      };

      secretKeyFile = mkOption {
        type = types.nullOr types.path;
        default = null;
        example = lib.literalExpression "config.sops.secrets.grafana-secret-key.path";
        description = ''
          File holding Grafana's `security.secret_key`. Loaded through systemd's
          `LoadCredential`, so the file only has to be readable by root at start time
          and never lands in the Nix store.

          Deliberately a plain path rather than a sops-nix integration: wire it up to
          whatever secret manager you already use.
        '';
      };

      dashboards = {
        node-exporter-full = dashboardOption "Provision grafana.com dashboard 1860 (Node Exporter Full).";
        smartctl-exporter = dashboardOption "Provision grafana.com dashboard 22604 (Smartctl Exporter).";
        zfs-pools = dashboardOption "Provision the ZFS pools dashboard. Needs the ZFS collector.";
        drive-health = dashboardOption "Provision the CrystalDiskInfo-style drive health dashboard. Needs the SMART collector and the smartctl recording rules.";
      };

      extraDashboards = mkOption {
        type = types.attrsOf types.path;
        default = { };
        example = lib.literalExpression ''{ my-dashboard = ./my-dashboard.json; }'';
        description = ''
          Extra dashboards to provision, as { name = path-to-json; }. These are passed
          through unmodified, so they must already reference the datasource UIDs
          "prometheus" and "loki".
        '';
      };
    };

    alerts = {
      enable = mkEnableOption ''
        Alertmanager, wired to Prometheus and delivering by email.

        The alerting rules themselves are plain PromQL in rules/alerts.yml, checked by
        promtool at build time like everything else here -- so a typo in an expression
        fails `nixos-rebuild`, not at 3am
      '';

      defaultRules = mkOption {
        type = types.bool;
        default = true;
        description = ''
          Load the alerting rules shipped with this module: drive failing or degraded,
          failed self-test, drive overheating, pool not ONLINE, unrepairable data
          errors, pool over 85% full, scrub or snapshots overdue, exporter down, and a
          collector that has stopped refreshing its file.

          Turn this off and point
          {option}`services.telemetry.server.prometheus.ruleFiles` at your own copy to
          change the thresholds.
        '';
      };

      to = mkOption {
        type = types.listOf types.str;
        example = [ "you@example.com" ];
        description = "Where notifications are sent.";
      };

      from = mkOption {
        type = types.str;
        example = "alerts@example.com";
        description = "Envelope sender. Many providers require this to be the account you authenticate as.";
      };

      smarthost = mkOption {
        type = types.str;
        example = "smtp.fastmail.com:587";
        description = "SMTP server and port to relay through.";
      };

      username = mkOption {
        type = types.nullOr types.str;
        default = null;
        description = "SMTP username. Null sends unauthenticated, which only works for a local relay.";
      };

      passwordFile = mkOption {
        type = types.nullOr types.path;
        default = null;
        example = lib.literalExpression "config.sops.secrets.smtp-password.path";
        description = ''
          File containing the SMTP password. Passed to Alertmanager as
          `smtp_auth_password_file` and read at runtime, so it never enters the Nix
          store. Must be readable by the alertmanager user.
        '';
      };

      requireTLS = mkOption {
        type = types.bool;
        default = true;
        description = "Require STARTTLS. Turn off only for a local relay on loopback.";
      };

      repeatInterval = mkOption {
        type = types.str;
        default = "24h";
        description = ''
          How long before an alert that is still firing is re-sent. A drive does not
          stop failing, so this is the knob that decides how often it nags you.
        '';
      };

      settings = mkOption {
        type = types.attrs;
        default = { };
        description = ''
          Merged over the generated Alertmanager configuration, last wins. The escape
          hatch for inhibit rules, extra receivers, or routing by severity.
        '';
      };
    };

    proxy = {
      enable = mkOption {
        type = types.bool;
        default = false;
        description = ''
          Run Caddy in front of Grafana. Off by default because a reverse proxy is
          usually something you already have; turn it on to get Grafana on port 80
          without exposing its own port.
        '';
      };

      virtualHost = mkOption {
        type = types.str;
        default = ":80";
        example = "metrics.example.com";
        description = ''
          Caddy virtual host. A bare hostname makes Caddy provision a certificate
          automatically; ":80" serves plain HTTP on every name.
        '';
      };
    };
  };

  config = mkIf cfg.enable {
    # Grafana uses this key to encrypt datasource credentials in its database, and
    # nixpkgs stopped shipping a default in 26.05. Caught here rather than left to
    # Grafana's own assertion, which names an option this module manages for you.
    assertions = [
      {
        assertion = !(cfg.grafana.generateSecretKey && cfg.grafana.secretKeyFile != null);
        message = ''
          services.telemetry.server.grafana.generateSecretKey and .secretKeyFile are
          mutually exclusive. Setting secretKeyFile turns generation off by default, so
          this only fires if you asked for both explicitly.
        '';
      }
      {
        assertion = !cfg.alerts.enable || cfg.prometheus.enable;
        message = ''
          services.telemetry.server.alerts.enable requires prometheus.enable -- the
          alerting rules are evaluated by Prometheus, which then pushes to
          Alertmanager.
        '';
      }
      {
        assertion = !cfg.grafana.enable || secretKeyPath != null;
        message = ''
          Grafana needs a secret key. Either leave
          services.telemetry.server.grafana.generateSecretKey at its default so one is
          generated on first start, or point .secretKeyFile at a file you manage.
        '';
      }
    ];

    # --------------------------------------------------------------------------------
    # Loki
    # --------------------------------------------------------------------------------
    services.loki = mkIf cfg.loki.enable {
      enable = true;
      configuration = {
        auth_enabled = false;

        server = {
          http_listen_port = ports.loki;
          # Explicit: the collectors have to reach this from another host or container.
          http_listen_address = cfg.listenAddress;
        };

        common = {
          path_prefix = cfg.loki.dataDir;
          storage.filesystem = {
            chunks_directory = "${cfg.loki.dataDir}/chunks";
            rules_directory = "${cfg.loki.dataDir}/rules";
          };
          ring = {
            instance_addr = "127.0.0.1";
            kvstore.store = "inmemory";
          };
          replication_factor = 1;
        };

        schema_config.configs = [{
          from = "2024-01-01";
          store = "tsdb";
          object_store = "filesystem";
          schema = "v13";
          index = { prefix = "index_"; period = "24h"; };
        }];
      };
    };

    # --------------------------------------------------------------------------------
    # Prometheus
    # --------------------------------------------------------------------------------
    services.prometheus = mkIf cfg.prometheus.enable {
      enable = true;
      port = ports.prometheus;
      listenAddress = cfg.listenAddress;
      retentionTime = cfg.prometheus.retentionTime;

      globalConfig = {
        scrape_interval = cfg.prometheus.scrapeInterval;
        evaluation_interval = cfg.prometheus.scrapeInterval;
        scrape_timeout = "10s";
      };

      ruleFiles = cfg.prometheus.ruleFiles
        ++ lib.optional (cfg.alerts.enable && cfg.alerts.defaultRules) ../rules/alerts.yml;

      scrapeConfigs =
        optionals (cfg.prometheus.nodeTargets != [ ]) [{
          job_name = "node";
          relabel_configs = hostLabelRelabel;
          static_configs = [{ targets = cfg.prometheus.nodeTargets; }];
        }]
        ++ optionals (cfg.prometheus.smartctlTargets != [ ]) [{
          job_name = "smartctl";
          scrape_interval = cfg.prometheus.smartctlScrapeInterval;
          scrape_timeout = "30s";
          relabel_configs = hostLabelRelabel;
          static_configs = [{ targets = cfg.prometheus.smartctlTargets; }];
        }]
        ++ cfg.prometheus.extraScrapeConfigs;

      # Alertmanager lives inside this block rather than in a `services.prometheus.*`
      # definition of its own: Nix will not let one attrset both define
      # `services.prometheus` wholesale and add a path underneath it.
      alertmanagers = mkIf cfg.alerts.enable [{
        static_configs = [{ targets = [ "127.0.0.1:${toString ports.alertmanager}" ]; }];
      }];

      alertmanager = mkIf cfg.alerts.enable {
        enable = true;
        port = ports.alertmanager;
        listenAddress = cfg.listenAddress;
        configuration = lib.recursiveUpdate
          {
            global = {
              smtp_smarthost = cfg.alerts.smarthost;
              smtp_from = cfg.alerts.from;
              smtp_require_tls = cfg.alerts.requireTLS;
            }
            // lib.optionalAttrs (cfg.alerts.username != null) {
              smtp_auth_username = cfg.alerts.username;
            }
            // lib.optionalAttrs (cfg.alerts.passwordFile != null) {
              smtp_auth_password_file = toString cfg.alerts.passwordFile;
            };

            route = {
              receiver = "email";
              # By alertname and host, so nine failing drives on one machine arrive as one
              # mail rather than nine.
              group_by = [ "alertname" "host" ];
              group_wait = "1m";
              group_interval = "10m";
              repeat_interval = cfg.alerts.repeatInterval;
            };

            receivers = [{
              name = "email";
              email_configs = map
                (addr: {
                  to = addr;
                  send_resolved = true;
                })
                cfg.alerts.to;
            }];
          }
          cfg.alerts.settings;
      };
    };

    # --------------------------------------------------------------------------------
    # Grafana
    # --------------------------------------------------------------------------------
    services.grafana = mkIf cfg.grafana.enable {
      enable = true;

      settings = {
        server = {
          http_addr = mkDefault cfg.listenAddress;
          http_port = mkDefault ports.grafana;
        };
      }
      # A file provider rather than a literal, so the key is never written into a
      # world-readable store path. The credential is mounted by LoadCredential below.
      // lib.optionalAttrs (secretKeyPath != null) {
        security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}";
      };

      provision = {
        enable = true;

        dashboards.settings.providers = [{
          name = "provisioned";
          disableDeletion = true;
          options = {
            path = "/etc/grafana-dashboards";
            foldersFromFilesStructure = true;
          };
        }];

        # Grafana cannot change the UID of an existing datasource in place. Its
        # provisioner finds the datasource by name, then calls UpdateDataSource, which
        # re-fetches using `id AND uid` -- with a new uid that row does not match, so it
        # fails with "data source not found" and Grafana refuses to start. Deleting
        # first turns the upsert into a plain insert; the provisioner runs deletes
        # before inserts for exactly this case and suppresses the deletion event because
        # the datasource is about to be re-created. Safe permanently: both datasources
        # are fully declarative and dashboards bind to them by UID, not database ID.
        datasources.settings.deleteDatasources =
          [{ name = "Prometheus"; orgId = 1; }]
          ++ optionals cfg.loki.enable [{ name = "Loki"; orgId = 1; }];

        datasources.settings.datasources =
          optionals cfg.prometheus.enable [{
            name = "Prometheus";
            type = "prometheus";
            uid = dsUid.prometheus;
            url = "http://127.0.0.1:${toString ports.prometheus}";
            isDefault = true;
            jsonData = {
              # What $__rate_interval is derived from. Leaving it unset makes Grafana
              # assume 15s and mis-size every rate() window. See scrapeInterval above.
              timeInterval = cfg.prometheus.scrapeInterval;
              httpMethod = "POST";
            };
          }]
          ++ optionals cfg.loki.enable [{
            name = "Loki";
            type = "loki";
            uid = dsUid.loki;
            url = "http://127.0.0.1:${toString ports.loki}";
          }];
      };
    };

    systemd.services.grafana = mkIf (cfg.grafana.enable && secretKeyPath != null) {
      serviceConfig.LoadCredential = [ "secret_key:${secretKeyPath}" ];
    };

    # Runs as the grafana user, into a directory users.users.grafana.createHome already
    # made and owns -- so there is no chown, and no fighting with Grafana over the
    # directory's ownership. UMask makes the file 0600 on creation; the explicit chmod
    # also repairs a file left too permissive by an earlier version or a manual copy.
    systemd.services.grafana-secret-key =
      mkIf (cfg.grafana.enable && cfg.grafana.generateSecretKey) {
        description = "Generate Grafana's secret key if it does not exist yet";
        wantedBy = [ "grafana.service" ];
        before = [ "grafana.service" ];
        serviceConfig = {
          Type = "oneshot";
          RemainAfterExit = true;
          User = "grafana";
          Group = "grafana";
          UMask = "0077";
        };
        script = ''
          key=${lib.escapeShellArg secretKeyPath}
          if [ ! -s "$key" ]; then
            ${pkgs.openssl}/bin/openssl rand -base64 32 > "$key"
          fi
          chmod 0600 "$key"
        '';
      };

    environment.etc = mkIf cfg.grafana.enable (
      mapAttrs' (name: src: nameValuePair "grafana-dashboards/${name}.json" { source = src; })
        dashboards
    );

    # --------------------------------------------------------------------------------
    # Reverse proxy
    # --------------------------------------------------------------------------------
    services.caddy = mkIf cfg.proxy.enable {
      enable = true;
      virtualHosts.${cfg.proxy.virtualHost}.extraConfig = ''
        reverse_proxy localhost:${toString ports.grafana}
      '';
    };

    # --------------------------------------------------------------------------------
    # Firewall
    # --------------------------------------------------------------------------------
    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall (
      optionals cfg.loki.enable [ ports.loki ]
      ++ optionals cfg.prometheus.enable [ ports.prometheus ]
      # The Alertmanager UI is how you silence an alert. Note anyone who can reach it
      # can silence anything -- same trust model as the Prometheus port above.
      ++ optionals cfg.alerts.enable [ ports.alertmanager ]
      ++ (if cfg.proxy.enable then [ 80 443 ] else optionals cfg.grafana.enable [ ports.grafana ])
    );
  };
}