1 files changed, 87 insertions, 0 deletions
diff --git a/hosts/poweredge-pro/ollama.nix b/hosts/poweredge-pro/ollama.nix
new file mode 100644
index 0000000..e660e24
--- /dev/null
+++ b/hosts/poweredge-pro/ollama.nix
@@ -0,0 +1,87 @@
+{ pkgs, ... }: {
+	containers.ollama = {
+		autoStart = true;
+		privateNetwork = true;
+		hostBridge = "br-lan1";
+		localMacAddress = "02:00:00:00:77:15";
+
+		# Host path
+		bindMounts = {
+			"/var/lib/ollama/models" = {
+				hostPath = "/VMDisk/models";
+				isReadOnly = false;
+			};
+		};
+
+		# GPU
+		allowedDevices = [
+			{ node = "/dev/nvidia0";           modifier = "rw"; }
+			{ node = "/dev/nvidia1";           modifier = "rw"; }
+			{ node = "/dev/nvidiactl";         modifier = "rw"; }
+			{ node = "/dev/nvidia-uvm";        modifier = "rw"; }
+			{ node = "/dev/nvidia-uvm-tools";  modifier = "rw"; }
+			{ node = "/dev/nvidia-modeset";    modifier = "rw"; }
+		];
+
+		bindMounts = {
+			# NVENC/NVDEC - video transcoding
+			"/dev/nvidia0"           = { hostPath = "/dev/nvidia0";           isReadOnly = false; };
+			"/dev/nvidia1"           = { hostPath = "/dev/nvidia1";           isReadOnly = false; };
+			"/dev/nvidiactl"         = { hostPath = "/dev/nvidiactl";         isReadOnly = false; };
+			# CUDA - required for ML inference
+			"/dev/nvidia-uvm"        = { hostPath = "/dev/nvidia-uvm";        isReadOnly = false; };
+			"/dev/nvidia-uvm-tools"  = { hostPath = "/dev/nvidia-uvm-tools";  isReadOnly = false; };
+			# Modeset - needed by some driver paths
+			"/dev/nvidia-modeset"    = { hostPath = "/dev/nvidia-modeset";    isReadOnly = false; };
+		};
+
+		config = { lib, pkg, config, ... }: {
+			imports = [ ./nvidia.nix ];
+
+			# Network
+			networking.interfaces.eth0.useDHCP = true;
+			networking.firewall.allowedTCPPorts = [ 80 11434 ];  # Caddy, ollama
+
+			# Ollama
+			services.ollama = {
+				enable = true;
+				# CUDA 13 dropped Pascal, so build against CUDA 12.
+				package = pkgs.ollama-cuda.override { cudaPackages = pkgs.cudaPackages_12; };
+				host = "0.0.0.0";
+				port = 11434;
+				environmentVariables = {
+					OLLAMA_SCHED_SPREAD = "1";  # spread models across both GPUs
+					OLLAMA_KEEP_ALIVE = "-1";
+					OLLAMA_HOST = "0.0.0.0:11434";
+					OLLAMA_CONTEXT_LENGTH = "65536";
+					OLLAMA_NUM_PARALLEL = "1";
+					# OLLAMA_FLASH_ATTENTION = "1";  # optional; test on your cards
+				};
+			};
+			systemd.services.ollama.serviceConfig.DynamicUser = lib.mkForce false;
+
+			# Web ui
+			services.open-webui = {
+				enable = true;
+				host = "127.0.0.1";
+				port = 8080;
+				environment = {
+					OLLAMA_BASE_URL = "http://127.0.0.1:11434";
+					ANONYMIZED_TELEMETRY = "False";
+					DO_NOT_TRACK = "True";
+					SCARF_NO_ANALYTICS = "True";
+				};
+			};
+
+			# Reverse proxy
+			services.caddy = {
+				enable = true;
+				virtualHosts.":80".extraConfig = ''
+					reverse_proxy localhost:8080
+				'';
+			};
+
+			system.stateVersion = "26.05";
+		};
+	};
+}