{ pkgs, config, ... }: {
	# Log shipping via Alloy
	services.alloy.enable = true;

	environment.etc."alloy/config.alloy".text = ''
		loki.source.journal "journal" {
			forward_to    = [loki.write.default.receiver]
			relabel_rules = loki.relabel.journal.rules
			max_age       = "12h"
		}

		loki.relabel "journal" {
			forward_to = []
			rule {
				source_labels = ["__journal__systemd_unit"]
				target_label  = "unit"
			}
		}

		loki.write "default" {
			endpoint {
				url = "http://lgtm:3100/loki/api/v1/push"
			}
		}
	'';

	# Export prometheus data from host for container to access
	services.prometheus.exporters.smartctl = {
		enable = true;
		port = 9633;
		listenAddress = "0.0.0.0";
	};

	services.prometheus.exporters.node = {
		enable = true;
		port = 9100;
		listenAddress = "0.0.0.0";
		enabledCollectors = [ "systemd" "diskstats" "filesystem" "meminfo" "cpu" ];
		# zfs + hwmon already on by default, no need to list them
	};

	# # Host config — zpool_influxdb needs real pool access
	#	services.telegraf = {
	#		enable = true;
	#		extraConfig = {
	#			agent.interval = "30s";
	#
	#			inputs.execd = [{
	#				command = [ "${pkgs.zpool_influxdb}/bin/zpool_influxdb" "--execd" ];
	#				signal = "none";
	#				data_format = "influx";
	#			}];
	#
	#			outputs.prometheus_client = [{
	#				listen = ":9273";
	#				metric_version = 2;
	#			}];
	#		};
	#	};

	# Allow ports for prometheus
	networking.firewall.allowedTCPPorts = [ 9100 9273 9633 ]; # at least reachable from container subnet

	# Secret key for grafana
	sops.secrets.grafana-secret-key = { sopsFile = ./resources/secrets/grafana.yaml; key = "secret_key"; };  # Office

	# Container
	containers.grafana = {
		autoStart = true;
		privateNetwork = true;
		hostBridge = "br-lan0";
		localMacAddress = "02:00:00:00:00:06";

		# Bind wg0-router secret to container
		bindMounts."/run/secrets/grafana-secret_key" = {
			hostPath = config.sops.secrets.grafana-secret-key.path;
			isReadOnly = true;
		};

		config = { lib, pkgs, config, ... }: {
			# Network
			networking.enableIPv6 = false;
			networking.interfaces.eth0.useDHCP = true;
			networking.firewall.allowedTCPPorts = [ 80 3100 9090 ];  # Caddy (grafana) + loki + prometheus

			# Loki
			services.loki = {
				enable = true;
				configuration = {
					server = {
						http_listen_port = 3100;
						http_listen_address = "0.0.0.0"; # explicit, since the host's Alloy needs to reach it to push logs
					};
					auth_enabled = false;

					common = {
						path_prefix = "/var/lib/loki";
						storage.filesystem = {
							chunks_directory = "/var/lib/loki/chunks";
							rules_directory = "/var/lib/loki/rules";
						};
						ring = {
							instance_addr = "127.0.0.1";
							kvstore.store = "inmemory";
						};
						replication_factor = 1;
					};

					schema_config.configs = [{
						from = "2024-01-01";
						store = "tsdb";
						object_store = "filesystem";
						schema = "v13";
						index = { prefix = "index_"; period = "24h"; };
					}];
				};
			};

			# Prometheus
			services.prometheus = {
				enable = true;
				scrapeConfigs = [
					{
						job_name = "smartctl";
						static_configs = [{ targets = [ "poweredge:9633" ]; }];
					}
					{
						job_name = "node";
						static_configs = [{ targets = [ "poweredge:9100" ]; }];
					}
					{
						job_name = "zpool";
						static_configs = [{ targets = [ "poweredge:9273" ]; }];
					}
				];
			};

			# Grafana
			# https://wiki.nixos.org/wiki/Grafana
			services.grafana = {
				enable = true;
				settings.server = { http_addr = "0.0.0.0"; http_port = 3000; };
				provision = {
					enable = true;

					# Creates a *mutable* dashboard provider, pulling from /etc/grafana-dashboards.
					# With this, you can manually provision dashboards from JSON with `environment.etc` like below.
					dashboards.settings.providers = [{
						name = "my dashboards";
						disableDeletion = true;
						options = {
							path = "/etc/grafana-dashboards";
							foldersFromFilesStructure = true;
						};
					}];

					datasources.settings.datasources = [
						{ name = "Prometheus"; type = "prometheus"; url = "http://127.0.0.1:9090"; isDefault = true; }
						{ name = "Loki"; type = "loki"; url = "http://127.0.0.1:3100"; }
					];
				};
			};

			# Grafana secret key
			systemd.services.grafana.serviceConfig = {
				LoadCredential = [
					"secret_key:/run/secrets/grafana-secret_key"
				];
			};
			services.grafana.settings.security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}";

			# Grafana dashboards
			environment.etc = let
				mkDashDl = id: rev: "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download";
			in {
				"grafana-dashboards/node-exporter-full.json".source = pkgs.fetchurl {
					url = mkDashDl 1860 45;
					sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY=";
				};
				"grafana-dashboards/smartctl-exporter.json".source = pkgs.fetchurl {
					url = mkDashDl 22604 2;
					sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88=";
				};
				"grafana-dashboards/zfs-pool-metrics.json".source = pkgs.fetchurl {
					url = mkDashDl 15362 1;
					sha256 = "sha256-latX1gwRjdpIalT5kpm9MfDaaXFLZCJ0ZP8CZJo8guI=";
				};
			};

			# Reverse proxy
			services.caddy = {
				enable = true;
				virtualHosts.":80".extraConfig = ''
					reverse_proxy localhost:3000
				'';
			};

			system.stateVersion = "26.05";
		};
	};
}