{ pkgs, ... }: { containers.ollama = { autoStart = true; privateNetwork = true; hostBridge = "br-lan1"; localMacAddress = "02:00:00:00:77:15"; # Host path bindMounts = { "/var/lib/ollama/models" = { hostPath = "/VMDisk/models"; isReadOnly = false; }; }; # GPU allowedDevices = [ { node = "/dev/nvidia0"; modifier = "rw"; } { node = "/dev/nvidia1"; modifier = "rw"; } { node = "/dev/nvidiactl"; modifier = "rw"; } { node = "/dev/nvidia-uvm"; modifier = "rw"; } { node = "/dev/nvidia-uvm-tools"; modifier = "rw"; } { node = "/dev/nvidia-modeset"; modifier = "rw"; } ]; bindMounts = { # NVENC/NVDEC - video transcoding "/dev/nvidia0" = { hostPath = "/dev/nvidia0"; isReadOnly = false; }; "/dev/nvidia1" = { hostPath = "/dev/nvidia1"; isReadOnly = false; }; "/dev/nvidiactl" = { hostPath = "/dev/nvidiactl"; isReadOnly = false; }; # CUDA - required for ML inference "/dev/nvidia-uvm" = { hostPath = "/dev/nvidia-uvm"; isReadOnly = false; }; "/dev/nvidia-uvm-tools" = { hostPath = "/dev/nvidia-uvm-tools"; isReadOnly = false; }; # Modeset - needed by some driver paths "/dev/nvidia-modeset" = { hostPath = "/dev/nvidia-modeset"; isReadOnly = false; }; }; config = { lib, pkg, config, ... }: { imports = [ ./nvidia.nix ]; # Network networking.interfaces.eth0.useDHCP = true; networking.firewall.allowedTCPPorts = [ 80 11434 ]; # Caddy, ollama # Ollama services.ollama = { enable = true; # CUDA 13 dropped Pascal, so build against CUDA 12. package = pkgs.ollama-cuda.override { cudaPackages = pkgs.cudaPackages_12; }; host = "0.0.0.0"; port = 11434; environmentVariables = { OLLAMA_SCHED_SPREAD = "1"; # spread models across both GPUs OLLAMA_KEEP_ALIVE = "-1"; OLLAMA_HOST = "0.0.0.0:11434"; OLLAMA_CONTEXT_LENGTH = "65536"; OLLAMA_NUM_PARALLEL = "1"; # OLLAMA_FLASH_ATTENTION = "1"; # optional; test on your cards }; }; systemd.services.ollama.serviceConfig.DynamicUser = lib.mkForce false; # Web ui services.open-webui = { enable = true; host = "127.0.0.1"; port = 8080; environment = { OLLAMA_BASE_URL = "http://127.0.0.1:11434"; ANONYMIZED_TELEMETRY = "False"; DO_NOT_TRACK = "True"; SCARF_NO_ANALYTICS = "True"; }; }; # Reverse proxy services.caddy = { enable = true; virtualHosts.":80".extraConfig = '' reverse_proxy localhost:8080 ''; }; system.stateVersion = "26.05"; }; }; }