graphana/lgtm stack container config

3 files changed, 219 insertions, 0 deletions

diff --git a/hosts/poweredge/configuration.nix b/hosts/poweredge/configuration.nix
index d503fd1..c15f222 100644
--- a/hosts/poweredge/configuration.nix
+++ b/hosts/poweredge/configuration.nix
@@ -7,8 +7,10 @@
 		./gitea.nix
 		./immich.nix
 		./jellyfin.nix
+		./lgtm.nix
 		./networking.nix
 		./nvidia.nix
+		./transmission.nix
 		#./notification-mailer.nix  # TODO move some of this stuff to archetype
 	];
 
diff --git a/hosts/poweredge/lgtm.nix b/hosts/poweredge/lgtm.nix
new file mode 100644
index 0000000..6f43c02
--- /dev/null
+++ b/hosts/poweredge/lgtm.nix
@@ -0,0 +1,201 @@
+{ pkgs, config, ... }: {
+	# Log shipping via Alloy
+	services.alloy.enable = true;
+
+	environment.etc."alloy/config.alloy".text = ''
+		loki.source.journal "journal" {
+			forward_to    = [loki.write.default.receiver]
+			relabel_rules = loki.relabel.journal.rules
+			max_age       = "12h"
+		}
+
+		loki.relabel "journal" {
+			forward_to = []
+			rule {
+				source_labels = ["__journal__systemd_unit"]
+				target_label  = "unit"
+			}
+		}
+
+		loki.write "default" {
+			endpoint {
+				url = "http://lgtm:3100/loki/api/v1/push"
+			}
+		}
+	'';
+
+	# Export prometheus data from host for container to access
+	services.prometheus.exporters.smartctl = {
+		enable = true;
+		port = 9633;
+		listenAddress = "0.0.0.0";
+	};
+
+	services.prometheus.exporters.node = {
+		enable = true;
+		port = 9100;
+		listenAddress = "0.0.0.0";
+		enabledCollectors = [ "systemd" "diskstats" "filesystem" "meminfo" "cpu" ];
+		# zfs + hwmon already on by default, no need to list them
+	};
+
+	# # Host config — zpool_influxdb needs real pool access
+	#	services.telegraf = {
+	#		enable = true;
+	#		extraConfig = {
+	#			agent.interval = "30s";
+	#
+	#			inputs.execd = [{
+	#				command = [ "${pkgs.zpool_influxdb}/bin/zpool_influxdb" "--execd" ];
+	#				signal = "none";
+	#				data_format = "influx";
+	#			}];
+	#
+	#			outputs.prometheus_client = [{
+	#				listen = ":9273";
+	#				metric_version = 2;
+	#			}];
+	#		};
+	#	};
+
+	# Allow ports for prometheus
+	networking.firewall.allowedTCPPorts = [ 9100 9273 9633 ]; # at least reachable from container subnet
+
+	# Secret key for grafana
+	sops.secrets.grafana-secret-key = { sopsFile = ./resources/secrets/grafana.yaml; key = "secret_key"; };  # Office
+
+	# Container
+	containers.grafana = {
+		autoStart = true;
+		privateNetwork = true;
+		hostBridge = "br-lan0";
+		localMacAddress = "02:00:00:00:00:06";
+
+		# Bind wg0-router secret to container
+		bindMounts."/run/secrets/grafana-secret_key" = {
+			hostPath = config.sops.secrets.grafana-secret-key.path;
+			isReadOnly = true;
+		};
+
+		config = { lib, pkgs, config, ... }: {
+			# Network
+			networking.enableIPv6 = false;
+			networking.interfaces.eth0.useDHCP = true;
+			networking.firewall.allowedTCPPorts = [ 80 3100 9090 ];  # Caddy (grafana) + loki + prometheus
+
+			# Loki
+			services.loki = {
+				enable = true;
+				configuration = {
+					server = {
+						http_listen_port = 3100;
+						http_listen_address = "0.0.0.0"; # explicit, since the host's Alloy needs to reach it to push logs
+					};
+					auth_enabled = false;
+
+					common = {
+						path_prefix = "/var/lib/loki";
+						storage.filesystem = {
+							chunks_directory = "/var/lib/loki/chunks";
+							rules_directory = "/var/lib/loki/rules";
+						};
+						ring = {
+							instance_addr = "127.0.0.1";
+							kvstore.store = "inmemory";
+						};
+						replication_factor = 1;
+					};
+
+					schema_config.configs = [{
+						from = "2024-01-01";
+						store = "tsdb";
+						object_store = "filesystem";
+						schema = "v13";
+						index = { prefix = "index_"; period = "24h"; };
+					}];
+				};
+			};
+
+			# Prometheus
+			services.prometheus = {
+				enable = true;
+				scrapeConfigs = [
+					{
+						job_name = "smartctl";
+						static_configs = [{ targets = [ "poweredge:9633" ]; }];
+					}
+					{
+						job_name = "node";
+						static_configs = [{ targets = [ "poweredge:9100" ]; }];
+					}
+					{
+						job_name = "zpool";
+						static_configs = [{ targets = [ "poweredge:9273" ]; }];
+					}
+				];
+			};
+
+			# Grafana
+			# https://wiki.nixos.org/wiki/Grafana
+			services.grafana = {
+				enable = true;
+				settings.server = { http_addr = "0.0.0.0"; http_port = 3000; };
+				provision = {
+					enable = true;
+
+					# Creates a *mutable* dashboard provider, pulling from /etc/grafana-dashboards.
+					# With this, you can manually provision dashboards from JSON with `environment.etc` like below.
+					dashboards.settings.providers = [{
+						name = "my dashboards";
+						disableDeletion = true;
+						options = {
+							path = "/etc/grafana-dashboards";
+							foldersFromFilesStructure = true;
+						};
+					}];
+
+					datasources.settings.datasources = [
+						{ name = "Prometheus"; type = "prometheus"; url = "http://127.0.0.1:9090"; isDefault = true; }
+						{ name = "Loki"; type = "loki"; url = "http://127.0.0.1:3100"; }
+					];
+				};
+			};
+
+			# Grafana secret key
+			systemd.services.grafana.serviceConfig = {
+				LoadCredential = [
+					"secret_key:/run/secrets/grafana-secret_key"
+				];
+			};
+			services.grafana.settings.security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}";
+
+			# Grafana dashboards
+			environment.etc = let
+				mkDashDl = id: rev: "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download";
+			in {
+				"grafana-dashboards/node-exporter-full.json".source = pkgs.fetchurl {
+					url = mkDashDl 1860 45;
+					sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY=";
+				};
+				"grafana-dashboards/smartctl-exporter.json".source = pkgs.fetchurl {
+					url = mkDashDl 22604 2;
+					sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88=";
+				};
+				"grafana-dashboards/zfs-pool-metrics.json".source = pkgs.fetchurl {
+					url = mkDashDl 15362 1;
+					sha256 = "sha256-latX1gwRjdpIalT5kpm9MfDaaXFLZCJ0ZP8CZJo8guI=";
+				};
+			};
+
+			# Reverse proxy
+			services.caddy = {
+				enable = true;
+				virtualHosts.":80".extraConfig = ''
+					reverse_proxy localhost:3000
+				'';
+			};
+
+			system.stateVersion = "26.05";
+		};
+	};
+}
diff --git a/hosts/poweredge/resources/secrets/grafana.yaml b/hosts/poweredge/resources/secrets/grafana.yaml
new file mode 100644
index 0000000..91b9188
--- /dev/null
+++ b/hosts/poweredge/resources/secrets/grafana.yaml
@@ -0,0 +1,16 @@
+secret_key: ENC[AES256_GCM,data:9GHuENHGk+VKukRuUJi+fxOX+ZFa5q5t5UiGtxSQOEctu9LOJqvHJerFv3U=,iv:6OJTeWbotDVj/ogg9WC170Q4Kdfd1PXf0hHwXd/YaUQ=,tag:qlnEl2uzDuhwOKp17R76Tg==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrc0tWWHhtM2Q5UC9weSsr
+            WDdaOXJjZGVoekJKaXUyUXhsRXMxMjQrYUFnCnBqTGJEMFF6clQ1RXd5NE9CenQ5
+            U0swNHdNUUxndnlycGVGM0lpY0dFaVEKLS0tIHVlK2MySUJzaGVXd21iWFFzcEJM
+            Nmh3V2hIbUhWNUxlOE1idTdWNWZ6alEKWae75mhvfpTNOcvm7x1zf19QkmZqrAf7
+            jpkyKDE//d4Rn3h4Ogpl5bdXb9RyOoSEJUG+EDPFyynqdodCJ2ucpw==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1zfvmt2avdlfz0fvchczplc84u7m8vqausm7zytl9s4x9m9yax4cqy30zpz
+    lastmodified: "2026-06-18T22:40:22Z"
+    mac: ENC[AES256_GCM,data:5HoPrQU8Q+SGEH4jQHy1otFXGILqttDpcbv+edNoKI/QFoUE74M4PrGzltlManZstqK3nfWV3HNIcY6GRe8TMP5ANC/8UIXfEDerP1Q0kj6xeyLumMCJ6iCjygF2SzL5YMWqBSsCJvH5O10sWncc2iPnGrQPJ8PksIxkXCe4YYk=,iv:f0WpJwENciWkyDKpGM/PaE2lmXWvNiMAxNpqlCEFhM8=,tag:nQAPTJPYsgRIUkDxktIvmA==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.1