diff options
Diffstat (limited to 'hosts')
| -rw-r--r-- | hosts/poweredge-pro/configuration.nix | 2 | ||||
| -rw-r--r-- | hosts/poweredge-pro/dhcp-hosts.nix | 1 | ||||
| -rw-r--r-- | hosts/poweredge-pro/nvidia.nix | 34 | ||||
| -rw-r--r-- | hosts/poweredge-pro/ollama.nix | 87 |
4 files changed, 124 insertions, 0 deletions
diff --git a/hosts/poweredge-pro/configuration.nix b/hosts/poweredge-pro/configuration.nix index f3ea1fb..7463cc5 100644 --- a/hosts/poweredge-pro/configuration.nix +++ b/hosts/poweredge-pro/configuration.nix @@ -7,6 +7,8 @@ ./fileshares.nix ./key.nix ./networking.nix + ./nvidia.nix + ./ollama.nix ./searxng.nix ]; diff --git a/hosts/poweredge-pro/dhcp-hosts.nix b/hosts/poweredge-pro/dhcp-hosts.nix index 08e3ebd..0a622f7 100644 --- a/hosts/poweredge-pro/dhcp-hosts.nix +++ b/hosts/poweredge-pro/dhcp-hosts.nix @@ -15,6 +15,7 @@ let filebrowser = { macAddress = "02:00:00:00:77:12"; staticIp = "192.168.77.12"; }; searxng = { macAddress = "02:00:00:00:77:13"; staticIp = "192.168.77.13"; }; embedtube = { macAddress = "02:00:00:00:77:14"; staticIp = "192.168.77.14"; }; + ollama = { macAddress = "02:00:00:00:77:15"; staticIp = "192.168.77.15"; }; WIN-BGBGC27B1PF = { macAddress = "d8:9e:f3:1c:f0:1c"; staticIp = "192.168.77.30"; }; # DACS DESKTOP-CQ4UIF9 = { macAddress = "08:bf:b8:3f:bd:48"; staticIp = "192.168.77.31"; }; # Kyle WIN-6FVMAFOID4G = { macAddress = "00:a0:98:58:10:85"; staticIp = "192.168.77.33"; }; # QB diff --git a/hosts/poweredge-pro/nvidia.nix b/hosts/poweredge-pro/nvidia.nix new file mode 100644 index 0000000..4f641b2 --- /dev/null +++ b/hosts/poweredge-pro/nvidia.nix @@ -0,0 +1,34 @@ +{ lib, config, pkgs, ... }: { + # Configure nixpkgs for nvidia/cuda + nixpkgs.config = { + #cudaSupport = true; + # Compile CUDA kernels only for the P100 (compute capability 6.0). + # The container reuses the host's pkgs, so this has to be set here. + cudaCapabilities = [ "6.0" ]; + allowUnfreePredicate = pkg: + pkgs._cuda.lib.allowUnfreeCudaPredicate pkg + || builtins.elem (lib.getName pkg) [ + "nvidia-x11" + "nvidia-kernel-modules" + "open-webui" + ]; + }; + + # NVIDIA + services.xserver.videoDrivers = [ "nvidia" ]; # xserver.videoDrivers does not imply X + hardware.graphics.enable = true; + hardware.nvidia = { + modesetting.enable = true; # Required + nvidiaSettings = false; # Don't need gui + open = false; + package = config.boot.kernelPackages.nvidiaPackages.legacy_580; # Support for P600 + powerManagement.enable = false; # Can cause bugs + powerManagement.finegrained = false; # Only works on turing and newer + }; + + # Packages + environment.systemPackages = with pkgs; [ + config.hardware.nvidia.package # nvidia-smi + nvtopPackages.nvidia + ]; +} diff --git a/hosts/poweredge-pro/ollama.nix b/hosts/poweredge-pro/ollama.nix new file mode 100644 index 0000000..e660e24 --- /dev/null +++ b/hosts/poweredge-pro/ollama.nix @@ -0,0 +1,87 @@ +{ pkgs, ... }: { + containers.ollama = { + autoStart = true; + privateNetwork = true; + hostBridge = "br-lan1"; + localMacAddress = "02:00:00:00:77:15"; + + # Host path + bindMounts = { + "/var/lib/ollama/models" = { + hostPath = "/VMDisk/models"; + isReadOnly = false; + }; + }; + + # GPU + allowedDevices = [ + { node = "/dev/nvidia0"; modifier = "rw"; } + { node = "/dev/nvidia1"; modifier = "rw"; } + { node = "/dev/nvidiactl"; modifier = "rw"; } + { node = "/dev/nvidia-uvm"; modifier = "rw"; } + { node = "/dev/nvidia-uvm-tools"; modifier = "rw"; } + { node = "/dev/nvidia-modeset"; modifier = "rw"; } + ]; + + bindMounts = { + # NVENC/NVDEC - video transcoding + "/dev/nvidia0" = { hostPath = "/dev/nvidia0"; isReadOnly = false; }; + "/dev/nvidia1" = { hostPath = "/dev/nvidia1"; isReadOnly = false; }; + "/dev/nvidiactl" = { hostPath = "/dev/nvidiactl"; isReadOnly = false; }; + # CUDA - required for ML inference + "/dev/nvidia-uvm" = { hostPath = "/dev/nvidia-uvm"; isReadOnly = false; }; + "/dev/nvidia-uvm-tools" = { hostPath = "/dev/nvidia-uvm-tools"; isReadOnly = false; }; + # Modeset - needed by some driver paths + "/dev/nvidia-modeset" = { hostPath = "/dev/nvidia-modeset"; isReadOnly = false; }; + }; + + config = { lib, pkg, config, ... }: { + imports = [ ./nvidia.nix ]; + + # Network + networking.interfaces.eth0.useDHCP = true; + networking.firewall.allowedTCPPorts = [ 80 11434 ]; # Caddy, ollama + + # Ollama + services.ollama = { + enable = true; + # CUDA 13 dropped Pascal, so build against CUDA 12. + package = pkgs.ollama-cuda.override { cudaPackages = pkgs.cudaPackages_12; }; + host = "0.0.0.0"; + port = 11434; + environmentVariables = { + OLLAMA_SCHED_SPREAD = "1"; # spread models across both GPUs + OLLAMA_KEEP_ALIVE = "-1"; + OLLAMA_HOST = "0.0.0.0:11434"; + OLLAMA_CONTEXT_LENGTH = "65536"; + OLLAMA_NUM_PARALLEL = "1"; + # OLLAMA_FLASH_ATTENTION = "1"; # optional; test on your cards + }; + }; + systemd.services.ollama.serviceConfig.DynamicUser = lib.mkForce false; + + # Web ui + services.open-webui = { + enable = true; + host = "127.0.0.1"; + port = 8080; + environment = { + OLLAMA_BASE_URL = "http://127.0.0.1:11434"; + ANONYMIZED_TELEMETRY = "False"; + DO_NOT_TRACK = "True"; + SCARF_NO_ANALYTICS = "True"; + }; + }; + + # Reverse proxy + services.caddy = { + enable = true; + virtualHosts.":80".extraConfig = '' + reverse_proxy localhost:8080 + ''; + }; + + system.stateVersion = "26.05"; + }; + }; +} |