From 805992ac3af22a3d1de7c57a96861d322aa062b3 Mon Sep 17 00:00:00 2001 From: Tim Keller Date: Fri, 30 Oct 2026 18:11:51 -0500 Subject: replace lgtm.nix with metrics container using telemetry-flake --- hosts/poweredge/lgtm.nix | 201 -------------------------------------------- hosts/poweredge/metrics.nix | 43 ++++++++++ 2 files changed, 43 insertions(+), 201 deletions(-) delete mode 100644 hosts/poweredge/lgtm.nix create mode 100644 hosts/poweredge/metrics.nix (limited to 'hosts') diff --git a/hosts/poweredge/lgtm.nix b/hosts/poweredge/lgtm.nix deleted file mode 100644 index 6f43c02..0000000 --- a/hosts/poweredge/lgtm.nix +++ /dev/null @@ -1,201 +0,0 @@ -{ pkgs, config, ... }: { - # Log shipping via Alloy - services.alloy.enable = true; - - environment.etc."alloy/config.alloy".text = '' - loki.source.journal "journal" { - forward_to = [loki.write.default.receiver] - relabel_rules = loki.relabel.journal.rules - max_age = "12h" - } - - loki.relabel "journal" { - forward_to = [] - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "unit" - } - } - - loki.write "default" { - endpoint { - url = "http://lgtm:3100/loki/api/v1/push" - } - } - ''; - - # Export prometheus data from host for container to access - services.prometheus.exporters.smartctl = { - enable = true; - port = 9633; - listenAddress = "0.0.0.0"; - }; - - services.prometheus.exporters.node = { - enable = true; - port = 9100; - listenAddress = "0.0.0.0"; - enabledCollectors = [ "systemd" "diskstats" "filesystem" "meminfo" "cpu" ]; - # zfs + hwmon already on by default, no need to list them - }; - - # # Host config — zpool_influxdb needs real pool access - # services.telegraf = { - # enable = true; - # extraConfig = { - # agent.interval = "30s"; - # - # inputs.execd = [{ - # command = [ "${pkgs.zpool_influxdb}/bin/zpool_influxdb" "--execd" ]; - # signal = "none"; - # data_format = "influx"; - # }]; - # - # outputs.prometheus_client = [{ - # listen = ":9273"; - # metric_version = 2; - # }]; - # }; - # }; - - # Allow ports for prometheus - networking.firewall.allowedTCPPorts = [ 9100 9273 9633 ]; # at least reachable from container subnet - - # Secret key for grafana - sops.secrets.grafana-secret-key = { sopsFile = ./resources/secrets/grafana.yaml; key = "secret_key"; }; # Office - - # Container - containers.grafana = { - autoStart = true; - privateNetwork = true; - hostBridge = "br-lan0"; - localMacAddress = "02:00:00:00:00:06"; - - # Bind wg0-router secret to container - bindMounts."/run/secrets/grafana-secret_key" = { - hostPath = config.sops.secrets.grafana-secret-key.path; - isReadOnly = true; - }; - - config = { lib, pkgs, config, ... }: { - # Network - networking.enableIPv6 = false; - networking.interfaces.eth0.useDHCP = true; - networking.firewall.allowedTCPPorts = [ 80 3100 9090 ]; # Caddy (grafana) + loki + prometheus - - # Loki - services.loki = { - enable = true; - configuration = { - server = { - http_listen_port = 3100; - http_listen_address = "0.0.0.0"; # explicit, since the host's Alloy needs to reach it to push logs - }; - auth_enabled = false; - - common = { - path_prefix = "/var/lib/loki"; - storage.filesystem = { - chunks_directory = "/var/lib/loki/chunks"; - rules_directory = "/var/lib/loki/rules"; - }; - ring = { - instance_addr = "127.0.0.1"; - kvstore.store = "inmemory"; - }; - replication_factor = 1; - }; - - schema_config.configs = [{ - from = "2024-01-01"; - store = "tsdb"; - object_store = "filesystem"; - schema = "v13"; - index = { prefix = "index_"; period = "24h"; }; - }]; - }; - }; - - # Prometheus - services.prometheus = { - enable = true; - scrapeConfigs = [ - { - job_name = "smartctl"; - static_configs = [{ targets = [ "poweredge:9633" ]; }]; - } - { - job_name = "node"; - static_configs = [{ targets = [ "poweredge:9100" ]; }]; - } - { - job_name = "zpool"; - static_configs = [{ targets = [ "poweredge:9273" ]; }]; - } - ]; - }; - - # Grafana - # https://wiki.nixos.org/wiki/Grafana - services.grafana = { - enable = true; - settings.server = { http_addr = "0.0.0.0"; http_port = 3000; }; - provision = { - enable = true; - - # Creates a *mutable* dashboard provider, pulling from /etc/grafana-dashboards. - # With this, you can manually provision dashboards from JSON with `environment.etc` like below. - dashboards.settings.providers = [{ - name = "my dashboards"; - disableDeletion = true; - options = { - path = "/etc/grafana-dashboards"; - foldersFromFilesStructure = true; - }; - }]; - - datasources.settings.datasources = [ - { name = "Prometheus"; type = "prometheus"; url = "http://127.0.0.1:9090"; isDefault = true; } - { name = "Loki"; type = "loki"; url = "http://127.0.0.1:3100"; } - ]; - }; - }; - - # Grafana secret key - systemd.services.grafana.serviceConfig = { - LoadCredential = [ - "secret_key:/run/secrets/grafana-secret_key" - ]; - }; - services.grafana.settings.security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}"; - - # Grafana dashboards - environment.etc = let - mkDashDl = id: rev: "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download"; - in { - "grafana-dashboards/node-exporter-full.json".source = pkgs.fetchurl { - url = mkDashDl 1860 45; - sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY="; - }; - "grafana-dashboards/smartctl-exporter.json".source = pkgs.fetchurl { - url = mkDashDl 22604 2; - sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88="; - }; - "grafana-dashboards/zfs-pool-metrics.json".source = pkgs.fetchurl { - url = mkDashDl 15362 1; - sha256 = "sha256-latX1gwRjdpIalT5kpm9MfDaaXFLZCJ0ZP8CZJo8guI="; - }; - }; - - # Reverse proxy - services.caddy = { - enable = true; - virtualHosts.":80".extraConfig = '' - reverse_proxy localhost:3000 - ''; - }; - - system.stateVersion = "26.05"; - }; - }; -} diff --git a/hosts/poweredge/metrics.nix b/hosts/poweredge/metrics.nix new file mode 100644 index 0000000..edf3da9 --- /dev/null +++ b/hosts/poweredge/metrics.nix @@ -0,0 +1,43 @@ +{ config, inputs, ... }: { + imports = [ inputs.telemetry.nixosModules.collectors ]; + + services.telemetry.collectors = { + enable = true; + logs.lokiUrl = "http://metrics:3100"; # The container + }; + + containers.metrics = { + autoStart = true; + privateNetwork = true; + hostBridge = "br-lan0"; + localMacAddress = "02:00:00:00:00:06"; + ephemeral = true; + + # The module reads whatever path you point secretKeyFile at. + bindMounts = { + "/var/lib/prometheus2" = { hostPath = "/nix/persist/metrics/prometheus2"; isReadOnly = false; }; + "/var/lib/loki" = { hostPath = "/nix/persist/metrics/loki"; isReadOnly = false; }; + "/var/lib/grafana" = { hostPath = "/nix/persist/metrics/grafana"; isReadOnly = false; }; + }; + + config = { ... }: { + imports = [ inputs.telemetry.nixosModules.server ]; + + networking.enableIPv6 = false; + networking.interfaces.eth0.useDHCP = true; + + services.telemetry.server = { + enable = true; + collectorHosts = [ "poweredge" ]; + grafana.generateSecretKey = true; + proxy.enable = true; + }; + + # Pin loki uid+gid + users.users.loki = { uid = 3100; group = "loki"; isSystemUser = true; }; + users.groups.loki.gid = 3100; + + system.stateVersion = "26.05"; + }; + }; +} -- cgit v1.3.1