# The web half: Prometheus, Loki, Grafana and an optional reverse proxy.
#
# Nothing here reads host hardware, so this is the part you can put in a container. It
# reaches the collectors over the network -- see `collectorHosts`.
{ config, lib, pkgs, ... }:

let
  inherit (lib) mkIf mkOption mkEnableOption mkDefault types optionals filterAttrs mapAttrs' nameValuePair;

  cfg = config.services.telemetry.server;
  ports = config.services.telemetry.ports;

  # Fixed, because the generated dashboards reference them literally. dashboards/panels.py
  # emits {"type": "prometheus", "uid": "prometheus"} on every panel, and the jq fixup in
  # dashboards.nix rewrites grafana.com's placeholders to the same values. Changing these
  # would silently break every panel, so they are not options.
  dsUid = { prometheus = "prometheus"; loki = "loki"; };

  # A `host` label carrying the machine name with the port stripped off.
  #
  # `instance` cannot do this job. The same machine is scraped twice on different ports
  # -- the node exporter on :9100 serves the textfile metrics (zfs_*, smart_selftest_*,
  # smart_drive_label) while smartctl_* comes from its own exporter on :9633 -- so one
  # machine has two `instance` values, and the recording rules join straight across that
  # boundary. `host` is equal on both, which is what makes the dashboards' host picker
  # and those joins work at the same time.
  #
  # Note the regex is fully anchored by Prometheus, and only handles host:port. A
  # bracketed IPv6 literal needs an explicit label via `extraScrapeConfigs` instead.
  hostLabelRelabel = [{
    source_labels = [ "__address__" ];
    regex = "([^:]+)(?::[0-9]+)?";
    target_label = "host";
    replacement = "$1";
  }];

  builtinDashboards = pkgs.callPackage ./dashboards.nix { inherit dsUid; };

  selected = filterAttrs (name: _: cfg.grafana.dashboards.${name} or false) builtinDashboards;

  dashboards = selected // cfg.grafana.extraDashboards;

  dashboardOption = description: mkOption {
    inherit description;
    type = types.bool;
    default = true;
  };
in
{
  imports = [ ./common.nix ];

  options.services.telemetry.server = {
    enable = mkEnableOption "the Prometheus/Loki/Grafana server";

    collectorHosts = mkOption {
      type = types.listOf types.str;
      default = [ "localhost" ];
      example = [ "poweredge" "optiplex" ];
      description = ''
        Machines running modules/collectors.nix. Each is scraped on both the node and
        smartctl ports from {option}`services.telemetry.ports`, so this is normally the
        only thing on the server side that has to know about the hosts.

        Each entry also becomes the value of the `host` label on everything scraped
        from that machine, which is what the dashboards' host picker selects on. Use
        the same spelling you would type in a browser; if you list an IP address, that
        is what the picker will show.

        Set {option}`services.telemetry.server.prometheus.nodeTargets` or
        {option}`services.telemetry.server.prometheus.smartctlTargets` to take over
        target generation entirely.
      '';
    };

    listenAddress = mkOption {
      type = types.str;
      default = "0.0.0.0";
      description = ''
        Address Loki, Prometheus and Grafana bind to. Loki has to be reachable from
        every host running the collectors, so this defaults to all interfaces.
      '';
    };

    openFirewall = mkOption {
      type = types.bool;
      default = true;
      description = ''
        Open the ports of whichever services are enabled: Loki (the collectors push to
        it), Prometheus (useful for checking scrape health directly), and either the
        proxy or Grafana.
      '';
    };

    prometheus = {
      enable = mkOption {
        type = types.bool;
        default = true;
        description = "Run Prometheus.";
      };

      scrapeInterval = mkOption {
        type = types.str;
        default = "15s";
        description = ''
          Global scrape interval, and the value handed to Grafana's Prometheus
          datasource as `timeInterval`.

          These two MUST agree. Grafana derives `$__rate_interval` from `timeInterval`,
          and if it guesses wrong then `rate()` windows are mis-sized and every
          rate-based panel intermittently renders "No data" -- which is exactly what
          Prometheus' own 1m default does to the Node Exporter Full dashboard. Setting
          it here sets both.
        '';
      };

      nodeTargets = mkOption {
        type = types.listOf types.str;
        default = map (host: "${host}:${toString ports.node}") cfg.collectorHosts;
        defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the node exporter port";
        description = "Targets for the `node` scrape job.";
      };

      smartctlTargets = mkOption {
        type = types.listOf types.str;
        default = map (host: "${host}:${toString ports.smartctl}") cfg.collectorHosts;
        defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the smartctl exporter port";
        description = "Targets for the `smartctl` scrape job. Empty disables the job.";
      };

      smartctlScrapeInterval = mkOption {
        type = types.str;
        default = "60s";
        description = ''
          Scrape interval for the smartctl job, which opts out of the global one:
          querying SMART is slow and the data barely moves. Safe because the drive
          dashboards use no `rate()` queries at all.
        '';
      };

      ruleFiles = mkOption {
        type = types.listOf types.path;
        default = [ ../rules/smartctl.yml ];
        defaultText = lib.literalExpression "[ ../rules/smartctl.yml ]";
        description = ''
          Recording rule files. The default computes the per-drive Good/Caution/Bad
          verdict and the row-title labels that the drive health dashboard displays, so
          drop it only if you also drop that dashboard.

          {option}`services.prometheus.checkConfig` runs promtool over these at build
          time, so a broken rule fails the build rather than the service.
        '';
      };

      extraScrapeConfigs = mkOption {
        type = types.listOf types.attrs;
        default = [ ];
        description = "Additional scrape jobs, appended to the generated ones.";
      };

      retentionTime = mkOption {
        type = types.str;
        default = "90d";
        description = "How long to keep samples.";
      };
    };

    loki = {
      enable = mkOption {
        type = types.bool;
        default = true;
        description = "Run Loki to receive journal logs from the collectors.";
      };

      dataDir = mkOption {
        type = types.path;
        default = "/var/lib/loki";
        description = "Where Loki stores its chunks and index.";
      };
    };

    grafana = {
      enable = mkOption {
        type = types.bool;
        default = true;
        description = "Run Grafana, provisioned with datasources and dashboards.";
      };

      secretKeyFile = mkOption {
        type = types.nullOr types.path;
        default = null;
        example = lib.literalExpression "config.sops.secrets.grafana-secret-key.path";
        description = ''
          File holding Grafana's `security.secret_key`. Loaded through systemd's
          `LoadCredential`, so the file only has to be readable by root at start time
          and never lands in the Nix store.

          Deliberately a plain path rather than a sops-nix integration: wire it up to
          whatever secret manager you already use.
        '';
      };

      dashboards = {
        node-exporter-full = dashboardOption "Provision grafana.com dashboard 1860 (Node Exporter Full).";
        smartctl-exporter = dashboardOption "Provision grafana.com dashboard 22604 (Smartctl Exporter).";
        zfs-pools = dashboardOption "Provision the ZFS pools dashboard. Needs the ZFS collector.";
        drive-health = dashboardOption "Provision the CrystalDiskInfo-style drive health dashboard. Needs the SMART collector and the smartctl recording rules.";
      };

      extraDashboards = mkOption {
        type = types.attrsOf types.path;
        default = { };
        example = lib.literalExpression ''{ my-dashboard = ./my-dashboard.json; }'';
        description = ''
          Extra dashboards to provision, as { name = path-to-json; }. These are passed
          through unmodified, so they must already reference the datasource UIDs
          "prometheus" and "loki".
        '';
      };
    };

    proxy = {
      enable = mkOption {
        type = types.bool;
        default = false;
        description = ''
          Run Caddy in front of Grafana. Off by default because a reverse proxy is
          usually something you already have; turn it on to get Grafana on port 80
          without exposing its own port.
        '';
      };

      virtualHost = mkOption {
        type = types.str;
        default = ":80";
        example = "metrics.example.com";
        description = ''
          Caddy virtual host. A bare hostname makes Caddy provision a certificate
          automatically; ":80" serves plain HTTP on every name.
        '';
      };
    };
  };

  config = mkIf cfg.enable {
    # Grafana uses this key to encrypt datasource credentials in its database, and
    # nixpkgs stopped shipping a default in 26.05. Caught here rather than left to
    # Grafana's own assertion, which names an option this module manages for you.
    assertions = [{
      assertion = !cfg.grafana.enable || cfg.grafana.secretKeyFile != null;
      message = ''
        services.telemetry.server.grafana.secretKeyFile must be set when Grafana is
        enabled. Generate a key (`openssl rand -base64 32`), store it with whatever
        secret manager you use, and point this option at the resulting file.
      '';
    }];

    # --------------------------------------------------------------------------------
    # Loki
    # --------------------------------------------------------------------------------
    services.loki = mkIf cfg.loki.enable {
      enable = true;
      configuration = {
        auth_enabled = false;

        server = {
          http_listen_port = ports.loki;
          # Explicit: the collectors have to reach this from another host or container.
          http_listen_address = cfg.listenAddress;
        };

        common = {
          path_prefix = cfg.loki.dataDir;
          storage.filesystem = {
            chunks_directory = "${cfg.loki.dataDir}/chunks";
            rules_directory = "${cfg.loki.dataDir}/rules";
          };
          ring = {
            instance_addr = "127.0.0.1";
            kvstore.store = "inmemory";
          };
          replication_factor = 1;
        };

        schema_config.configs = [{
          from = "2024-01-01";
          store = "tsdb";
          object_store = "filesystem";
          schema = "v13";
          index = { prefix = "index_"; period = "24h"; };
        }];
      };
    };

    # --------------------------------------------------------------------------------
    # Prometheus
    # --------------------------------------------------------------------------------
    services.prometheus = mkIf cfg.prometheus.enable {
      enable = true;
      port = ports.prometheus;
      listenAddress = cfg.listenAddress;
      retentionTime = cfg.prometheus.retentionTime;

      globalConfig = {
        scrape_interval = cfg.prometheus.scrapeInterval;
        evaluation_interval = cfg.prometheus.scrapeInterval;
        scrape_timeout = "10s";
      };

      inherit (cfg.prometheus) ruleFiles;

      scrapeConfigs =
        optionals (cfg.prometheus.nodeTargets != [ ]) [{
          job_name = "node";
          relabel_configs = hostLabelRelabel;
          static_configs = [{ targets = cfg.prometheus.nodeTargets; }];
        }]
        ++ optionals (cfg.prometheus.smartctlTargets != [ ]) [{
          job_name = "smartctl";
          scrape_interval = cfg.prometheus.smartctlScrapeInterval;
          scrape_timeout = "30s";
          relabel_configs = hostLabelRelabel;
          static_configs = [{ targets = cfg.prometheus.smartctlTargets; }];
        }]
        ++ cfg.prometheus.extraScrapeConfigs;
    };

    # --------------------------------------------------------------------------------
    # Grafana
    # --------------------------------------------------------------------------------
    services.grafana = mkIf cfg.grafana.enable {
      enable = true;

      settings = {
        server = {
          http_addr = mkDefault cfg.listenAddress;
          http_port = mkDefault ports.grafana;
        };
      }
      # A file provider rather than a literal, so the key is never written into a
      # world-readable store path. The credential is mounted by LoadCredential below.
      // lib.optionalAttrs (cfg.grafana.secretKeyFile != null) {
        security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}";
      };

      provision = {
        enable = true;

        dashboards.settings.providers = [{
          name = "provisioned";
          disableDeletion = true;
          options = {
            path = "/etc/grafana-dashboards";
            foldersFromFilesStructure = true;
          };
        }];

        # Grafana cannot change the UID of an existing datasource in place. Its
        # provisioner finds the datasource by name, then calls UpdateDataSource, which
        # re-fetches using `id AND uid` -- with a new uid that row does not match, so it
        # fails with "data source not found" and Grafana refuses to start. Deleting
        # first turns the upsert into a plain insert; the provisioner runs deletes
        # before inserts for exactly this case and suppresses the deletion event because
        # the datasource is about to be re-created. Safe permanently: both datasources
        # are fully declarative and dashboards bind to them by UID, not database ID.
        datasources.settings.deleteDatasources =
          [{ name = "Prometheus"; orgId = 1; }]
          ++ optionals cfg.loki.enable [{ name = "Loki"; orgId = 1; }];

        datasources.settings.datasources =
          optionals cfg.prometheus.enable [{
            name = "Prometheus";
            type = "prometheus";
            uid = dsUid.prometheus;
            url = "http://127.0.0.1:${toString ports.prometheus}";
            isDefault = true;
            jsonData = {
              # What $__rate_interval is derived from. Leaving it unset makes Grafana
              # assume 15s and mis-size every rate() window. See scrapeInterval above.
              timeInterval = cfg.prometheus.scrapeInterval;
              httpMethod = "POST";
            };
          }]
          ++ optionals cfg.loki.enable [{
            name = "Loki";
            type = "loki";
            uid = dsUid.loki;
            url = "http://127.0.0.1:${toString ports.loki}";
          }];
      };
    };

    systemd.services.grafana = mkIf (cfg.grafana.enable && cfg.grafana.secretKeyFile != null) {
      serviceConfig.LoadCredential = [ "secret_key:${cfg.grafana.secretKeyFile}" ];
    };

    environment.etc = mkIf cfg.grafana.enable (
      mapAttrs' (name: src: nameValuePair "grafana-dashboards/${name}.json" { source = src; })
        dashboards
    );

    # --------------------------------------------------------------------------------
    # Reverse proxy
    # --------------------------------------------------------------------------------
    services.caddy = mkIf cfg.proxy.enable {
      enable = true;
      virtualHosts.${cfg.proxy.virtualHost}.extraConfig = ''
        reverse_proxy localhost:${toString ports.grafana}
      '';
    };

    # --------------------------------------------------------------------------------
    # Firewall
    # --------------------------------------------------------------------------------
    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall (
      optionals cfg.loki.enable [ ports.loki ]
      ++ optionals cfg.prometheus.enable [ ports.prometheus ]
      ++ (if cfg.proxy.enable then [ 80 443 ] else optionals cfg.grafana.enable [ ports.grafana ])
    );
  };
}