diff options
Diffstat (limited to 'hosts/poweredge/lgtm.nix')
| -rw-r--r-- | hosts/poweredge/lgtm.nix | 201 |
1 files changed, 201 insertions, 0 deletions
diff --git a/hosts/poweredge/lgtm.nix b/hosts/poweredge/lgtm.nix new file mode 100644 index 0000000..6f43c02 --- /dev/null +++ b/hosts/poweredge/lgtm.nix @@ -0,0 +1,201 @@ +{ pkgs, config, ... }: { + # Log shipping via Alloy + services.alloy.enable = true; + + environment.etc."alloy/config.alloy".text = '' + loki.source.journal "journal" { + forward_to = [loki.write.default.receiver] + relabel_rules = loki.relabel.journal.rules + max_age = "12h" + } + + loki.relabel "journal" { + forward_to = [] + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + } + + loki.write "default" { + endpoint { + url = "http://lgtm:3100/loki/api/v1/push" + } + } + ''; + + # Export prometheus data from host for container to access + services.prometheus.exporters.smartctl = { + enable = true; + port = 9633; + listenAddress = "0.0.0.0"; + }; + + services.prometheus.exporters.node = { + enable = true; + port = 9100; + listenAddress = "0.0.0.0"; + enabledCollectors = [ "systemd" "diskstats" "filesystem" "meminfo" "cpu" ]; + # zfs + hwmon already on by default, no need to list them + }; + + # # Host config — zpool_influxdb needs real pool access + # services.telegraf = { + # enable = true; + # extraConfig = { + # agent.interval = "30s"; + # + # inputs.execd = [{ + # command = [ "${pkgs.zpool_influxdb}/bin/zpool_influxdb" "--execd" ]; + # signal = "none"; + # data_format = "influx"; + # }]; + # + # outputs.prometheus_client = [{ + # listen = ":9273"; + # metric_version = 2; + # }]; + # }; + # }; + + # Allow ports for prometheus + networking.firewall.allowedTCPPorts = [ 9100 9273 9633 ]; # at least reachable from container subnet + + # Secret key for grafana + sops.secrets.grafana-secret-key = { sopsFile = ./resources/secrets/grafana.yaml; key = "secret_key"; }; # Office + + # Container + containers.grafana = { + autoStart = true; + privateNetwork = true; + hostBridge = "br-lan0"; + localMacAddress = "02:00:00:00:00:06"; + + # Bind wg0-router secret to container + bindMounts."/run/secrets/grafana-secret_key" = { + hostPath = config.sops.secrets.grafana-secret-key.path; + isReadOnly = true; + }; + + config = { lib, pkgs, config, ... }: { + # Network + networking.enableIPv6 = false; + networking.interfaces.eth0.useDHCP = true; + networking.firewall.allowedTCPPorts = [ 80 3100 9090 ]; # Caddy (grafana) + loki + prometheus + + # Loki + services.loki = { + enable = true; + configuration = { + server = { + http_listen_port = 3100; + http_listen_address = "0.0.0.0"; # explicit, since the host's Alloy needs to reach it to push logs + }; + auth_enabled = false; + + common = { + path_prefix = "/var/lib/loki"; + storage.filesystem = { + chunks_directory = "/var/lib/loki/chunks"; + rules_directory = "/var/lib/loki/rules"; + }; + ring = { + instance_addr = "127.0.0.1"; + kvstore.store = "inmemory"; + }; + replication_factor = 1; + }; + + schema_config.configs = [{ + from = "2024-01-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { prefix = "index_"; period = "24h"; }; + }]; + }; + }; + + # Prometheus + services.prometheus = { + enable = true; + scrapeConfigs = [ + { + job_name = "smartctl"; + static_configs = [{ targets = [ "poweredge:9633" ]; }]; + } + { + job_name = "node"; + static_configs = [{ targets = [ "poweredge:9100" ]; }]; + } + { + job_name = "zpool"; + static_configs = [{ targets = [ "poweredge:9273" ]; }]; + } + ]; + }; + + # Grafana + # https://wiki.nixos.org/wiki/Grafana + services.grafana = { + enable = true; + settings.server = { http_addr = "0.0.0.0"; http_port = 3000; }; + provision = { + enable = true; + + # Creates a *mutable* dashboard provider, pulling from /etc/grafana-dashboards. + # With this, you can manually provision dashboards from JSON with `environment.etc` like below. + dashboards.settings.providers = [{ + name = "my dashboards"; + disableDeletion = true; + options = { + path = "/etc/grafana-dashboards"; + foldersFromFilesStructure = true; + }; + }]; + + datasources.settings.datasources = [ + { name = "Prometheus"; type = "prometheus"; url = "http://127.0.0.1:9090"; isDefault = true; } + { name = "Loki"; type = "loki"; url = "http://127.0.0.1:3100"; } + ]; + }; + }; + + # Grafana secret key + systemd.services.grafana.serviceConfig = { + LoadCredential = [ + "secret_key:/run/secrets/grafana-secret_key" + ]; + }; + services.grafana.settings.security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}"; + + # Grafana dashboards + environment.etc = let + mkDashDl = id: rev: "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download"; + in { + "grafana-dashboards/node-exporter-full.json".source = pkgs.fetchurl { + url = mkDashDl 1860 45; + sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY="; + }; + "grafana-dashboards/smartctl-exporter.json".source = pkgs.fetchurl { + url = mkDashDl 22604 2; + sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88="; + }; + "grafana-dashboards/zfs-pool-metrics.json".source = pkgs.fetchurl { + url = mkDashDl 15362 1; + sha256 = "sha256-latX1gwRjdpIalT5kpm9MfDaaXFLZCJ0ZP8CZJo8guI="; + }; + }; + + # Reverse proxy + services.caddy = { + enable = true; + virtualHosts.":80".extraConfig = '' + reverse_proxy localhost:3000 + ''; + }; + + system.stateVersion = "26.05"; + }; + }; +} |