blob: e660e2490cabcb4718eda2f51d7bb29315a39eb1 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
|
{ pkgs, ... }: {
containers.ollama = {
autoStart = true;
privateNetwork = true;
hostBridge = "br-lan1";
localMacAddress = "02:00:00:00:77:15";
# Host path
bindMounts = {
"/var/lib/ollama/models" = {
hostPath = "/VMDisk/models";
isReadOnly = false;
};
};
# GPU
allowedDevices = [
{ node = "/dev/nvidia0"; modifier = "rw"; }
{ node = "/dev/nvidia1"; modifier = "rw"; }
{ node = "/dev/nvidiactl"; modifier = "rw"; }
{ node = "/dev/nvidia-uvm"; modifier = "rw"; }
{ node = "/dev/nvidia-uvm-tools"; modifier = "rw"; }
{ node = "/dev/nvidia-modeset"; modifier = "rw"; }
];
bindMounts = {
# NVENC/NVDEC - video transcoding
"/dev/nvidia0" = { hostPath = "/dev/nvidia0"; isReadOnly = false; };
"/dev/nvidia1" = { hostPath = "/dev/nvidia1"; isReadOnly = false; };
"/dev/nvidiactl" = { hostPath = "/dev/nvidiactl"; isReadOnly = false; };
# CUDA - required for ML inference
"/dev/nvidia-uvm" = { hostPath = "/dev/nvidia-uvm"; isReadOnly = false; };
"/dev/nvidia-uvm-tools" = { hostPath = "/dev/nvidia-uvm-tools"; isReadOnly = false; };
# Modeset - needed by some driver paths
"/dev/nvidia-modeset" = { hostPath = "/dev/nvidia-modeset"; isReadOnly = false; };
};
config = { lib, pkg, config, ... }: {
imports = [ ./nvidia.nix ];
# Network
networking.interfaces.eth0.useDHCP = true;
networking.firewall.allowedTCPPorts = [ 80 11434 ]; # Caddy, ollama
# Ollama
services.ollama = {
enable = true;
# CUDA 13 dropped Pascal, so build against CUDA 12.
package = pkgs.ollama-cuda.override { cudaPackages = pkgs.cudaPackages_12; };
host = "0.0.0.0";
port = 11434;
environmentVariables = {
OLLAMA_SCHED_SPREAD = "1"; # spread models across both GPUs
OLLAMA_KEEP_ALIVE = "-1";
OLLAMA_HOST = "0.0.0.0:11434";
OLLAMA_CONTEXT_LENGTH = "65536";
OLLAMA_NUM_PARALLEL = "1";
# OLLAMA_FLASH_ATTENTION = "1"; # optional; test on your cards
};
};
systemd.services.ollama.serviceConfig.DynamicUser = lib.mkForce false;
# Web ui
services.open-webui = {
enable = true;
host = "127.0.0.1";
port = 8080;
environment = {
OLLAMA_BASE_URL = "http://127.0.0.1:11434";
ANONYMIZED_TELEMETRY = "False";
DO_NOT_TRACK = "True";
SCARF_NO_ANALYTICS = "True";
};
};
# Reverse proxy
services.caddy = {
enable = true;
virtualHosts.":80".extraConfig = ''
reverse_proxy localhost:8080
'';
};
system.stateVersion = "26.05";
};
};
}
|
