From d2d1386abb358fa547a40fc8df613d799155b56d Mon Sep 17 00:00:00 2001
From: Tim Keller <tjk@tjkeller.xyz>
Date: Thu, 27 Aug 2026 12:45:44 -0500
Subject: split into a flake with separable collectors/server modules

Replaces the single lgtm.nix with a flake exposing two NixOS modules under
services.telemetry, so the web services can move into a container declared in
the consuming config rather than one hardcoded here.

  modules/collectors.nix  host-side: node + smartctl exporters, the ZFS and
                          SMART textfile collector timers, Alloy journal
                          shipping. Must run on the host -- reads /proc, /sys,
                          raw block devices and ZFS ioctls.
  modules/server.nix      Prometheus, Loki, Grafana, optional Caddy.
  modules/common.nix      ports, shared by both halves.
  modules/dashboards.nix  jq fixup and grafana.com fetches.
  modules/packages.nix    the two collectors as ordinary packages.

Modules are exported as paths, not functions, so they evaluate with the
consumer's nixpkgs; the flake's own input only affects packages/checks/devShells.

Multi-host support. collectorHosts is a list, both scrape jobs relabel
__address__ into a `host` label, and all 133 queries in the two generated
dashboards are scoped to host=~"$host" behind a single-select Host picker.
`instance` could not serve as that key: one machine is scraped twice on
different ports (textfile metrics on :9100, smartctl_* on :9633) and the
recording rules join across that boundary.

Grafana's secret key becomes a plain secretKeyFile path wired through
LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No
sops-nix input.

nix flake check runs promtool over the rules, diffs the committed dashboard
JSON against what the generators produce, and evaluates both modules.

Also fixes smart-selftest emitting a placeholder identity line when smartctl
cannot open a device. smartctl exits 0 with well-formed but empty JSON in that
case, so "SSD - unknown model - no serial" was reaching a dashboard row title;
it now emits nothing and lets the smartctl:ident fallback apply.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ
---
 README.md                    | 280 +++++++++++++++++++++++
 collectors/smart-selftest.py |  18 +-
 dashboards/gen_smart.py      |  92 ++++----
 dashboards/gen_zfs.py        |  76 +++---
 dashboards/smart.json        | 149 ++++++------
 dashboards/zfs-pools.json    | 173 ++++++++------
 flake.lock                   |  27 +++
 flake.nix                    | 109 +++++++++
 lgtm.nix                     | 534 -------------------------------------------
 modules/collectors.nix       | 368 +++++++++++++++++++++++++++++
 modules/common.nix           |  46 ++++
 modules/dashboards.nix       |  75 ++++++
 modules/default.nix          |  13 ++
 modules/packages.nix         |  45 ++++
 modules/server.nix           | 420 ++++++++++++++++++++++++++++++++++
 rules/smartctl.yml           |  38 +--
 16 files changed, 1692 insertions(+), 771 deletions(-)
 create mode 100644 README.md
 create mode 100644 flake.lock
 create mode 100644 flake.nix
 delete mode 100644 lgtm.nix
 create mode 100644 modules/collectors.nix
 create mode 100644 modules/common.nix
 create mode 100644 modules/dashboards.nix
 create mode 100644 modules/default.nix
 create mode 100644 modules/packages.nix
 create mode 100644 modules/server.nix

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..7bcea7f
--- /dev/null
+++ b/README.md
@@ -0,0 +1,280 @@
+# telemetry
+
+Host telemetry collection and a Prometheus/Loki/Grafana server, as two NixOS modules you
+can deploy together or apart.
+
+---
+
+## ⚠️ This repository is vibe coded. Run it at your own risk.
+
+Every line here — the modules, the collectors, the dashboard generators, the recording
+rules, and this README — was written by an LLM in conversation. No human has reviewed it
+line by line. There is no warranty of any kind. **Read it before you deploy it**, and pay
+particular attention to the systemd hardening options, the firewall rules, and anything
+touching raw block devices.
+
+What *has* been checked:
+
+- every dashboard query executed against a real Prometheus fed fixture data
+- `promtool` over the recording rules and the generated scrape config
+- `nix flake check`: both modules evaluate, and the committed dashboard JSON matches what
+  the generators produce
+- the collectors run against mock `smartctl`/`zpool`/`zfs`, and partially against real
+  hardware
+
+What has **not**:
+
+- no panel has been rendered in a real Grafana
+- no output has come from a real ZFS pool
+- no real SAS self-test log has been parsed
+
+Treat the numbers on the dashboards as unverified until you have sanity-checked them
+against `zpool status` and `smartctl -x` yourself.
+
+---
+
+## What it is
+
+The split is not cosmetic. The collectors read host hardware and kernel state — `/proc`,
+`/sys`, SMART over raw block devices, ZFS ioctls — which a container has no view of. The
+server reads none of that, so it can live wherever you like.
+
+| module | contains | runs where |
+|---|---|---|
+| `nixosModules.collectors` | node + smartctl exporters, ZFS and SMART textfile collectors, Alloy | the host, necessarily |
+| `nixosModules.server` | Prometheus, Loki, Grafana, optional Caddy | anywhere that can reach the collectors |
+| `nixosModules.default` | both | one machine |
+
+Grafana comes provisioned with four dashboards: Node Exporter Full (1860), Smartctl
+Exporter (22604), and two generated here — ZFS Pools and Drive Health, a
+CrystalDiskInfo-style per-drive view.
+
+```nix
+{
+  inputs.telemetry.url = "path:/home/timmy/docs/src/flakes/lgtm";
+}
+```
+
+The examples below take `inputs` as a module argument, which means passing it through:
+
+```nix
+nixosConfigurations.poweredge = nixpkgs.lib.nixosSystem {
+  specialArgs = { inherit inputs; };
+  # ...
+};
+```
+
+---
+
+## Example: one machine
+
+```nix
+{ inputs, ... }:
+{
+  imports = [ inputs.telemetry.nixosModules.default ];
+
+  services.telemetry = {
+    collectors.enable = true;
+    server = {
+      enable = true;
+      grafana.secretKeyFile = "/run/secrets/grafana-secret-key";
+      proxy.enable = true;                     # Grafana on :80
+    };
+  };
+}
+```
+
+Grafana at `http://localhost`. `collectorHosts` defaults to `[ "localhost" ]`.
+
+## Example: web services in a container
+
+The host runs the collectors; the container runs everything else. This is the intended
+shape.
+
+```nix
+{ config, inputs, ... }:
+{
+  imports = [ inputs.telemetry.nixosModules.collectors ];
+
+  services.telemetry.collectors = {
+    enable = true;
+    logs.lokiUrl = "http://metrics:3100";      # the container, by name
+  };
+
+  sops.secrets.grafana-secret-key = {
+    sopsFile = ./secrets/grafana.yaml;
+    key = "secret_key";
+  };
+
+  containers.metrics = {
+    autoStart = true;
+    privateNetwork = true;
+    hostBridge = "br-lan0";
+    localMacAddress = "02:00:00:00:00:06";
+
+    # The module reads whatever path you point secretKeyFile at.
+    bindMounts."/run/grafana-secret-key" = {
+      hostPath = config.sops.secrets.grafana-secret-key.path;
+      isReadOnly = true;
+    };
+
+    config = { ... }: {
+      imports = [ inputs.telemetry.nixosModules.server ];
+
+      networking.enableIPv6 = false;
+      networking.interfaces.eth0.useDHCP = true;
+
+      services.telemetry.server = {
+        enable = true;
+        collectorHosts = [ "poweredge" ];       # the host above
+        grafana.secretKeyFile = "/run/grafana-secret-key";
+        proxy.enable = true;
+      };
+
+      system.stateVersion = "26.05";            # the module does not set this for you
+    };
+  };
+}
+```
+
+Two names have to resolve for this to work: the container must be able to look up
+`poweredge`, and the host must be able to look up `metrics`. With `hostBridge` plus DHCP
+on the LAN that is your router's job; otherwise use IP addresses in both places.
+
+## Example: several machines
+
+List them, and import the collectors module on each:
+
+```nix
+# in the container
+services.telemetry.server.collectorHosts = [ "poweredge" "optiplex" ];
+```
+
+```nix
+# on optiplex
+{ inputs, ... }:
+{
+  imports = [ inputs.telemetry.nixosModules.collectors ];
+
+  services.telemetry.collectors = {
+    enable = true;
+    logs.lokiUrl = "http://metrics:3100";
+  };
+}
+```
+
+You get **one set of dashboards with a Host dropdown**, not a set per machine. Pick a
+host and the whole page swaps to it.
+
+| dashboard | picker(s) |
+|---|---|
+| Drive Health | **Host** → SATA drive / SAS drive |
+| ZFS Pools | **Host** → Pool |
+| Node Exporter Full | Job → Nodename → Instance |
+| Smartctl Exporter | Node |
+
+Dependent dropdowns repopulate on switch, so you never see poweredge's `tank` while
+viewing optiplex. Hosts appear and disappear on their own — the pickers are
+`label_values(..., host)`, so a machine shows up as soon as Prometheus scrapes it.
+
+The Host picker is **single-select with no "All"** on purpose: drive and pool names are
+not unique across a fleet, and two hosts at once would make `max by (device)` silently
+merge two different physical drives into one plausible-looking row.
+
+## Example: turning things off
+
+```nix
+services.telemetry = {
+  collectors = {
+    enable = true;
+    zfs.enable = false;                        # no ZFS on this box
+    logs.enable = false;                       # no journal shipping
+    node.extraCollectors = [ "interrupts" ];   # fills 1860's IRQ panel; high cardinality
+  };
+
+  server = {
+    enable = true;
+    loki.enable = false;
+    grafana.dashboards.zfs-pools = false;
+    grafana.extraDashboards.mine = ./my-dashboard.json;
+    prometheus.extraScrapeConfigs = [{
+      job_name = "router";
+      static_configs = [{ targets = [ "router:9100" ]; }];
+    }];
+  };
+};
+```
+
+Ports live in `services.telemetry.ports` (`node`, `smartctl`, `loki`, `prometheus`,
+`grafana`) and are shared by both halves.
+
+Jobs added through `extraScrapeConfigs` are passed to Prometheus untouched — they get no
+`host` label, so they will not show up in the dashboards' Host picker. Add your own
+`relabel_configs` if you want them to.
+
+---
+
+## Layout
+
+```
+flake.nix               modules, collector packages, checks, dev shell
+modules/
+  common.nix            ports, shared by both halves
+  collectors.nix        host-side exporters and timers
+  server.nix            Prometheus, Loki, Grafana, proxy
+  dashboards.nix        builds dashboard JSON into a provisionable form
+  packages.nix          the two collectors as ordinary packages
+collectors/*.py         the collectors themselves; runnable by hand
+dashboards/gen_*.py     dashboard generators
+dashboards/*.json       generated, committed, checked
+rules/smartctl.yml      Prometheus recording rules
+```
+
+## Working on it
+
+Dashboard JSON is **generated** — edit `dashboards/gen_*.py`, never the JSON:
+
+```sh
+python3 dashboards/gen_zfs.py   > dashboards/zfs-pools.json
+python3 dashboards/gen_smart.py > dashboards/smart.json
+```
+
+`nix flake check` fails if you forget.
+
+The collectors are plain scripts taking `--output PATH`, so you can see exactly what a
+machine reports:
+
+```sh
+nix run .#zfs-metrics         -- --output /tmp/zfs.prom
+sudo nix run .#smart-selftest -- --output /tmp/smart.prom
+```
+
+`smart-selftest` needs privileges to open the devices. Without them smartctl still exits
+0 with well-formed but empty JSON; the collector logs `no identity data` rather than
+inventing placeholder values.
+
+## Things that will bite you
+
+- **`prometheus.scrapeInterval` deliberately sets two things.** Grafana derives
+  `$__rate_interval` from the datasource's `timeInterval`; if that disagrees with the real
+  scrape interval, `rate()` windows are mis-sized and rate-based panels intermittently
+  render "No data". Don't configure the Grafana datasource by hand.
+- **The datasource UIDs `prometheus` and `loki` are fixed, not options.** The generated
+  dashboards reference them literally.
+- **Grafana's secret key is mandatory** as of nixpkgs 26.05. `openssl rand -base64 32`,
+  store it somewhere, point `grafana.secretKeyFile` at it.
+- **Changing `collectorHosts` changes the label set.** Panels look empty for a minute
+  after first deploy while the recording rules re-evaluate; data from before the change
+  stays under the old labels.
+- **smartd does not run self-tests just because it is enabled.** NixOS's default `-a`
+  flags contain no `-s` schedule, so self-test history stays empty:
+  ```nix
+  services.smartd.defaults.autodetected = "-a -o on -s (S/../.././02|L/../../6/03)";
+  ```
+- **SAS drives have no power-on-hours attribute.** smartctl sources `power_on_time` for
+  SCSI from the Background Scan Results log page, which is not the drive's lifetime meter
+  and often reads far too low. Drive Health takes the larger of that and the lifetime
+  stamp in the drive's own self-test log — so it can exceed what `smartctl -x` prints.
+- **IPv6 literals need help.** The `host` label is built by stripping the port off
+  `__address__` with `([^:]+)(?::[0-9]+)?`, which a bracketed IPv6 address does not match.
+  Use `prometheus.extraScrapeConfigs` with an explicit label for those.
diff --git a/collectors/smart-selftest.py b/collectors/smart-selftest.py
index fa9d68f..6cb026c 100644
--- a/collectors/smart-selftest.py
+++ b/collectors/smart-selftest.py
@@ -231,13 +231,25 @@ def collect_identity(reg: Registry, name: str, data: dict) -> None:
     This exists to be read by a human in a Grafana row title, so it is deliberately a
     single rendered string rather than separate labels -- Grafana can interpolate a
     variable's display text into a title, but cannot assemble one from several labels.
+
+    Emits nothing unless smartctl actually returned identifying data. When it cannot
+    open the device -- no privileges, a controller that needs an explicit -d type -- it
+    still exits 0 with a well-formed JSON document that simply has no identity fields,
+    and filling those in with placeholders would put "unknown model · no serial" in a
+    dashboard row title. Emitting nothing instead lets the `smartctl:ident` recording
+    rule fall back to "identity pending", which is the truth.
     """
     model = (data.get("model_name") or data.get("scsi_model_name")
-             or (data.get("scsi_product") or "") or "unknown model")
-    serial = data.get("serial_number") or "no serial"
+             or data.get("scsi_product") or "")
+    serial = data.get("serial_number") or ""
     capacity = human_capacity((data.get("user_capacity") or {}).get("bytes"))
 
-    parts = [p for p in (capacity, technology(data), model, serial) if p]
+    if not (model or serial or capacity):
+        warn(f"{name}: no identity data; is smartctl able to open the device?")
+        return
+
+    parts = [p for p in (capacity, technology(data), model or "unknown model",
+                         serial or "no serial") if p]
     reg.add("smart_drive_label", 1, {"device": name, "label": " · ".join(parts)},
             "Drive identity rendered for display; the value is always 1.")
 
diff --git a/dashboards/gen_smart.py b/dashboards/gen_smart.py
index 2cc0568..de9ab34 100644
--- a/dashboards/gen_smart.py
+++ b/dashboards/gen_smart.py
@@ -54,8 +54,14 @@ PASS_MAP = [{
     },
 }]
 
-ATA = 'device=~"$ata_device"'
-SAS = 'device=~"$sas_device"'
+# Every query on this dashboard is scoped to exactly one machine. The host variable is
+# deliberately single-select with no "All": `sda` is not a unique name across a fleet,
+# and the `max by (device)` reductions and joinByField transformations below would
+# silently merge two different physical drives into one row if more than one host were
+# ever selected at a time.
+HOST = 'host=~"$host"'
+ATA = f'{HOST}, device=~"$ata_device"'
+SAS = f'{HOST}, device=~"$sas_device"'
 
 # Splits "sde · GOOD · 6.0 TB · SAS 7.2K · HGST HUS726060AL5210 · NAHPJS45" into a
 # selectable value (the leading device name) and display text (the whole line). The
@@ -80,7 +86,7 @@ def power_on(sel=""):
     source is larger is therefore never an over-estimate, and recovers a sane number on
     drives whose power-on counter under-counts or was reset.
     """
-    braces = f"{{{sel}}}" if sel else ""
+    braces = "{" + ", ".join(x for x in (HOST, sel) if x) + "}"
     return (f'max by (device) (smartctl_device_power_on_seconds{braces} '
             f'or (smart_selftest_last_lifetime_hours{braces} * 3600))')
 
@@ -124,14 +130,14 @@ panels = []
 panels.append(row("Fleet Overview", 0))
 
 panels.append(stat(
-    "Healthy", [('count(smartctl:health_code == 0) or vector(0)', "drives")],
+    "Healthy", [(f'count(smartctl:health_code{{{HOST}}} == 0) or vector(0)', "drives")],
     gp(0, 1, 4, 4), colormode="background",
     thresholds={"mode": "absolute", "steps": [{"color": "green", "value": None}]},
     desc="Drives with no failing attribute, no SCSI uncorrected errors and a passing "
          "SMART self-assessment.",
 ))
 panels.append(stat(
-    "Caution", [('count(smartctl:health_code == 1) or vector(0)', "drives")],
+    "Caution", [(f'count(smartctl:health_code{{{HOST}}} == 1) or vector(0)', "drives")],
     gp(4, 1, 4, 4), colormode="background",
     thresholds={"mode": "absolute", "steps": [
         {"color": "green", "value": None}, {"color": "orange", "value": 1}]},
@@ -139,18 +145,19 @@ panels.append(stat(
          "defect list on SAS. Not failing yet — worth watching the trend.",
 ))
 panels.append(stat(
-    "Failing", [('count(smartctl:health_code == 2) or vector(0)', "drives")],
+    "Failing", [(f'count(smartctl:health_code{{{HOST}}} == 2) or vector(0)', "drives")],
     gp(8, 1, 4, 4), colormode="background", thresholds=OK_BAD,
     desc="SMART self-assessment failed, an attribute is at or below its threshold, or "
          "the drive reported errors it could not correct. Replace these.",
 ))
 panels.append(stat(
-    "Drives", [('count(smartctl_device_smart_status)', "total")],
+    "Drives", [(f'count(smartctl_device_smart_status{{{HOST}}})', "total")],
     gp(12, 1, 4, 4),
 ))
 panels.append(stat(
     "Hottest Drive",
-    [('max(smartctl_device_temperature{temperature_type="current"})', "max")],
+    [(f'max(smartctl_device_temperature{{{HOST}, temperature_type="current"}})',
+      "max")],
     gp(16, 1, 4, 4), unit="celsius", colormode="background",
     thresholds={"mode": "absolute", "steps": [
         {"color": "green", "value": None},
@@ -167,12 +174,12 @@ panels.append(table(
     "All Drives", [
         # group_left carries the descriptive labels from the info metric onto the
         # verdict, so this one query supplies device, model, serial and protocol.
-        ('smartctl:health_code * on(device) group_left(model_name, serial_number, '
-         'protocol) smartctl_device', None),
-        ('max by (device) (smartctl_device_temperature{temperature_type="current"})',
-         None),
+        (f'smartctl:health_code{{{HOST}}} * on(device) group_left(model_name, '
+         f'serial_number, protocol) smartctl_device{{{HOST}}}', None),
+        (f'max by (device) (smartctl_device_temperature{{{HOST}, '
+         f'temperature_type="current"}})', None),
         (power_on(), None),
-        ('max by (device) (smartctl_device_capacity_bytes)', None),
+        (f'max by (device) (smartctl_device_capacity_bytes{{{HOST}}})', None),
     ],
     gp(0, 5, 16, 10), keep="device|model_name|serial_number|protocol|Value.*",
     renames={
@@ -204,7 +211,8 @@ panels.append(table(
 
 panels.append(timeseries(
     "Temperatures",
-    [('smartctl_device_temperature{temperature_type="current"}', "{{device}}")],
+    [(f'smartctl_device_temperature{{{HOST}, temperature_type="current"}}',
+      "{{device}}")],
     gp(16, 5, 8, 10), unit="celsius",
     desc="Sustained operation above ~45C measurably shortens drive life.",
 ))
@@ -418,22 +426,23 @@ panels.append(row("Self-Tests", 53))
 
 panels.append(stat(
     "Drives Never Tested",
-    [('count(smartctl_device_smart_status) - '
-      'count(count by (device) (smart_selftest_last_lifetime_hours)) or '
-      'count(smartctl_device_smart_status)', "drives")],
+    [(f'count(smartctl_device_smart_status{{{HOST}}}) - '
+      f'count(count by (device) (smart_selftest_last_lifetime_hours{{{HOST}}})) or '
+      f'count(smartctl_device_smart_status{{{HOST}}})', "drives")],
     gp(0, 54, 4, 5), thresholds=OK_BAD, colormode="background",
     desc="Drives with no self-test log entries at all. smartd only runs tests if its "
          "config includes an -s schedule; merely enabling the service does not.",
 ))
 panels.append(stat(
     "Failed Self-Tests",
-    [('count(smart_selftest_last_passed == 0) or vector(0)', "tests")],
+    [(f'count(smart_selftest_last_passed{{{HOST}}} == 0) or vector(0)', "tests")],
     gp(4, 54, 4, 5), thresholds=OK_BAD, colormode="background",
 ))
 panels.append(stat(
     "Oldest Long Test",
     [(f'max({power_on()} / 3600 - on(device) group_right() '
-      f'smart_selftest_last_lifetime_hours{{test_type="extended"}})', "hours ago")],
+      f'smart_selftest_last_lifetime_hours{{{HOST}, test_type="extended"}})',
+      "hours ago")],
     gp(8, 54, 4, 5), unit="h",
     thresholds={"mode": "absolute", "steps": [
         {"color": "green", "value": None},
@@ -461,14 +470,14 @@ def _keyed(expr):
 panels.append(table(
     "Self-Test History", [
         # Only this query keeps device and test_type as their own columns.
-        ('label_join(smart_selftest_last_passed, "drive_test", "/", "device", '
-         '"test_type")', None),
-        (_keyed('smart_selftest_last_lifetime_hours'), None),
-        (_keyed('smart_selftest_count'), None),
+        (f'label_join(smart_selftest_last_passed{{{HOST}}}, "drive_test", "/", '
+         f'"device", "test_type")', None),
+        (_keyed(f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None),
+        (_keyed(f'smart_selftest_count{{{HOST}}}'), None),
         # Powered-on hours elapsed since that test ran. group_right() keeps the
         # right-hand side's labels, so device and test_type survive the subtraction.
         (_keyed(power_on() + ' / 3600 - on(device) group_right() '
-                'smart_selftest_last_lifetime_hours'), None),
+                f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None),
     ],
     gp(12, 54, 12, 10), keep="drive_test|device|test_type|Value.*",
     renames={
@@ -501,22 +510,22 @@ panels.append(table(
 panels.append(row("Attribute Trends", 64, collapsed=True, children=[
     timeseries(
         "Reallocated Sectors (all SATA drives)",
-        [('smartctl_device_attribute{attribute_id="5", attribute_value_type="raw"}',
-          "{{device}}")],
+        [(f'smartctl_device_attribute{{{HOST}, attribute_id="5", '
+          f'attribute_value_type="raw"}}', "{{device}}")],
         gp(0, 65, 8, 7), minval=0,
         custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
     ),
     timeseries(
         "Pending & Uncorrectable Sectors",
-        [('smartctl_device_attribute{attribute_id=~"197|198", '
-          'attribute_value_type="raw"}', "{{device}} {{attribute_name}}")],
+        [(f'smartctl_device_attribute{{{HOST}, attribute_id=~"197|198", '
+          f'attribute_value_type="raw"}}', "{{device}} {{attribute_name}}")],
         gp(8, 65, 8, 7), minval=0,
         custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
     ),
     timeseries(
         "CRC Errors (cabling)",
-        [('smartctl_device_attribute{attribute_id="199", attribute_value_type="raw"}',
-          "{{device}}")],
+        [(f'smartctl_device_attribute{{{HOST}, attribute_id="199", '
+          f'attribute_value_type="raw"}}', "{{device}}")],
         gp(16, 65, 8, 7), minval=0,
         custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"},
         desc="Attribute 199 counts errors on the SATA link rather than the platters. "
@@ -524,14 +533,14 @@ panels.append(row("Attribute Trends", 64, collapsed=True, children=[
     ),
     timeseries(
         "Data Written / Read (SAS)", [
-            ('smartctl_device_bytes_written', "{{device}} written"),
-            ('smartctl_device_bytes_read', "{{device}} read"),
+            (f'smartctl_device_bytes_written{{{HOST}}}', "{{device}} written"),
+            (f'smartctl_device_bytes_read{{{HOST}}}', "{{device}} read"),
         ],
         gp(0, 72, 12, 7), unit="bytes", minval=0,
     ),
     timeseries(
         "Health Code History",
-        [('smartctl:health_code', "{{device}}")],
+        [(f'smartctl:health_code{{{HOST}}}', "{{device}}")],
         gp(12, 72, 12, 7), mappings=HEALTH_MAP, minval=0, maxval=2,
         custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"},
     ),
@@ -540,7 +549,7 @@ panels.append(row("Attribute Trends", 64, collapsed=True, children=[
 panels.append(row("Collector Health", 65, collapsed=True, children=[
     timeseries(
         "smartctl Exit Status",
-        [('smartctl_device_smartctl_exit_status', "{{device}}")],
+        [(f'smartctl_device_smartctl_exit_status{{{HOST}}}', "{{device}}")],
         gp(0, 66, 8, 7), minval=0,
         desc="Non-zero means smartctl itself had trouble reading the device. Bit 0 is "
              "a command-line problem, bit 1 a device-open failure, bit 3 a failing "
@@ -548,13 +557,13 @@ panels.append(row("Collector Health", 65, collapsed=True, children=[
     ),
     timeseries(
         "Self-Test Collector Duration",
-        [('smart_selftest_collection_duration_seconds', "collection")],
+        [(f'smart_selftest_collection_duration_seconds{{{HOST}}}', "collection")],
         gp(8, 66, 8, 7), unit="s", minval=0,
     ),
     timeseries(
         "Devices Seen", [
-            ('smartctl_devices', "exporter"),
-            ('smart_selftest_devices', "self-test collector"),
+            (f'smartctl_devices{{{HOST}}}', "exporter"),
+            (f'smart_selftest_devices{{{HOST}}}', "self-test collector"),
         ],
         gp(16, 66, 8, 7), minval=0,
         desc="These should agree. A divergence means one of the two cannot see some "
@@ -575,9 +584,14 @@ print(json.dumps(dashboard(
         # `device=~"$ata_device"` selector below is unaffected, while the *text* the UI
         # shows -- and that ${ata_device:text} puts in the row title -- is the whole
         # line. That is what makes a collapsed row identifiable without opening it.
-        query_var("ata_device", 'label_values(smartctl:row{bus="ata"}, row)',
+        # One machine at a time. Not multi-select and no "All" -- see the comment
+        # on HOST above for why selecting two hosts at once would silently merge
+        # same-named drives rather than showing both.
+        query_var("host", "label_values(smartctl_device_smart_status, host)",
+                  label="Host", multi=False, include_all=False),
+        query_var("ata_device", 'label_values(smartctl:row{bus="ata", host=~"$host"}, row)',
                   label="SATA drive", regex=ROW_REGEX),
-        query_var("sas_device", 'label_values(smartctl:row{bus="sas"}, row)',
+        query_var("sas_device", 'label_values(smartctl:row{bus="sas", host=~"$host"}, row)',
                   label="SAS drive", regex=ROW_REGEX),
     ],
     tags=["smart", "storage", "hardware"],
diff --git a/dashboards/gen_zfs.py b/dashboards/gen_zfs.py
index 4edb38b..0057506 100644
--- a/dashboards/gen_zfs.py
+++ b/dashboards/gen_zfs.py
@@ -20,7 +20,8 @@ If you add a panel, add the metric to the collector first and keep this list hon
 
 import json
 
-from panels import (DS, OK_BAD, bargauge, gauge, gp, row, stat, table, timeseries)
+from panels import (DS, OK_BAD, bargauge, gauge, gp, query_var, row, stat, table,
+                    timeseries)
 
 
 STATE_MAP = [{
@@ -46,7 +47,12 @@ BOOL_MAP = [{
     },
 }]
 
-POOL = 'pool=~"$pool"'
+# Every query on this dashboard is scoped to exactly one machine. The host variable is
+# single-select with no "All" on purpose: pool names are not unique across a fleet, and
+# the `max by (pool)` reductions and joinByField transformations below would silently
+# merge two different pools that happen to share a name.
+HOST = 'host=~"$host"'
+POOL = f'{HOST}, pool=~"$pool"'
 panels = []
 
 # ======================================================================================
@@ -124,7 +130,7 @@ panels.append(stat(
 panels.append(stat(
     "Snapshots Overdue",
     [(f'count((time() - zfs_snapshot_newest_timestamp_seconds{{{POOL}}}) > on(label) '
-      f'group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)', "overdue")],
+      f'group_left() (2 * zfs_snapshot_interval_seconds{{{HOST}}})) or vector(0)', "overdue")],
     gp(15, 5, 4, 4), thresholds=OK_BAD, colormode="background",
     desc="Dataset/label pairs whose newest snapshot is more than two intervals old — "
          "i.e. the auto-snapshot timer has missed at least one run.",
@@ -294,7 +300,7 @@ panels.append(bargauge(
 panels.append(bargauge(
     "Snapshots vs Retention Target",
     [(f'min by (label) (zfs_snapshot_count{{{POOL}}} / on(label) group_left() '
-      f'zfs_snapshot_retention_target)', "{{label}}")],
+      f'zfs_snapshot_retention_target{{{HOST}}})', "{{label}}")],
     gp(8, 43, 8, 8), unit="percentunit", maxv=1.2,
     thresholds={"mode": "absolute", "steps": [
         {"color": "red", "value": None},
@@ -329,15 +335,16 @@ panels.append(timeseries(
 
 panels.append(stat(
     "Snapshot / Scrub Job Failures",
-    [('sum(node_systemd_unit_state{name=~"zfs-(snapshot|scrub).*",state="failed"}) '
-      'or vector(0)', "failed units")],
+    [(f'sum(node_systemd_unit_state{{{HOST}, name=~"zfs-(snapshot|scrub).*", '
+      f'state="failed"}}) or vector(0)', "failed units")],
     gp(0, 59, 6, 6), thresholds=OK_BAD, colormode="background",
     desc="systemd units for the auto-snapshot and auto-scrub timers currently in a "
          "failed state.",
 ))
 panels.append(bargauge(
     "Time Since Timer Last Fired",
-    [('time() - node_systemd_timer_last_trigger_seconds{name=~"zfs-.*"}', "{{name}}")],
+    [(f'time() - node_systemd_timer_last_trigger_seconds{{{HOST}, '
+      f'name=~"zfs-.*"}}', "{{name}}")],
     gp(6, 59, 18, 6), unit="s",
     desc="When each zfs-* systemd timer last triggered. This is the job side of the "
          "picture; the snapshot ages above are the result side.",
@@ -445,9 +452,9 @@ panels.append(row("Performance", 91))
 
 panels.append(timeseries(
     "Dataset Throughput", [
-        (f'rate(node_zfs_zpool_dataset_nread{{zpool=~"$pool"}}[$__rate_interval])',
+        (f'rate(node_zfs_zpool_dataset_nread{{{HOST}, zpool=~"$pool"}}[$__rate_interval])',
          "{{zpool}}/{{dataset}} read"),
-        (f'-rate(node_zfs_zpool_dataset_nwritten{{zpool=~"$pool"}}[$__rate_interval])',
+        (f'-rate(node_zfs_zpool_dataset_nwritten{{{HOST}, zpool=~"$pool"}}[$__rate_interval])',
          "{{zpool}}/{{dataset}} write"),
     ],
     gp(0, 92, 12, 8), unit="Bps",
@@ -456,9 +463,9 @@ panels.append(timeseries(
 ))
 panels.append(timeseries(
     "Dataset IOPS", [
-        (f'rate(node_zfs_zpool_dataset_reads{{zpool=~"$pool"}}[$__rate_interval])',
+        (f'rate(node_zfs_zpool_dataset_reads{{{HOST}, zpool=~"$pool"}}[$__rate_interval])',
          "{{zpool}}/{{dataset}} read"),
-        (f'-rate(node_zfs_zpool_dataset_writes{{zpool=~"$pool"}}[$__rate_interval])',
+        (f'-rate(node_zfs_zpool_dataset_writes{{{HOST}, zpool=~"$pool"}}[$__rate_interval])',
          "{{zpool}}/{{dataset}} write"),
     ],
     gp(12, 92, 12, 8), unit="iops",
@@ -467,24 +474,24 @@ panels.append(timeseries(
 
 panels.append(timeseries(
     "ARC Size", [
-        ("node_zfs_arc_size", "size"),
-        ("node_zfs_arc_c_max", "target max (c_max)"),
-        ("node_zfs_arc_c_min", "target min (c_min)"),
+        (f"node_zfs_arc_size{{{HOST}}}", "size"),
+        (f"node_zfs_arc_c_max{{{HOST}}}", "target max (c_max)"),
+        (f"node_zfs_arc_c_min{{{HOST}}}", "target min (c_min)"),
     ],
     gp(0, 100, 8, 8), unit="bytes", minval=0,
 ))
 panels.append(timeseries(
     "ARC Hit Ratio",
-    [("rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min("
-      "rate(node_zfs_arc_hits[$__rate_interval]) + "
-      "rate(node_zfs_arc_misses[$__rate_interval]), 1)", "hit ratio")],
+    [(f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval]) / clamp_min("
+      f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval]) + "
+      f"rate(node_zfs_arc_misses{{{HOST}}}[$__rate_interval]), 1)", "hit ratio")],
     gp(8, 100, 8, 8), unit="percentunit", minval=0,
     desc="clamp_min avoids a divide-by-zero while the pool is idle.",
 ))
 panels.append(timeseries(
     "ARC Hits / Misses", [
-        ("rate(node_zfs_arc_hits[$__rate_interval])", "hits"),
-        ("rate(node_zfs_arc_misses[$__rate_interval])", "misses"),
+        (f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval])", "hits"),
+        (f"rate(node_zfs_arc_misses{{{HOST}}}[$__rate_interval])", "misses"),
     ],
     gp(16, 100, 8, 8), unit="ops", minval=0,
 ))
@@ -553,7 +560,7 @@ panels.append(row("Pool Internals", 108, collapsed=True, children=[
 panels.append(row("Collector Health", 109, collapsed=True, children=[
     timeseries(
         "ZFS Collector Duration",
-        [("zfs_metrics_collection_duration_seconds", "collection")],
+        [(f"zfs_metrics_collection_duration_seconds{{{HOST}}}", "collection")],
         gp(0, 110, 8, 7), unit="s", minval=0,
         desc="How long collectors/zfs-metrics.py takes. Listing every snapshot is the "
              "slow part; if this approaches the 1 minute timer interval, raise "
@@ -561,7 +568,7 @@ panels.append(row("Collector Health", 109, collapsed=True, children=[
     ),
     timeseries(
         "Textfile Age",
-        [("time() - node_textfile_mtime_seconds", "{{file}}")],
+        [(f"time() - node_textfile_mtime_seconds{{{HOST}}}", "{{file}}")],
         gp(8, 110, 8, 7), unit="s", minval=0,
         desc="Age of each .prom file. If this climbs without bound the collector has "
              "stopped writing and every ZFS metric above is stale.",
@@ -569,7 +576,7 @@ panels.append(row("Collector Health", 109, collapsed=True, children=[
     timeseries(
         "Scrape Health", [
             ('up{job="node"}', "node exporter up"),
-            ("node_textfile_scrape_error", "textfile parse error"),
+            (f"node_textfile_scrape_error{{{HOST}}}", "textfile parse error"),
         ],
         gp(16, 110, 8, 7), minval=0,
         desc="node_textfile_scrape_error goes to 1 if a .prom file is malformed, in "
@@ -591,21 +598,16 @@ dashboard = {
     "graphTooltip": 1,
     "refresh": "1m",
     "time": {"from": "now-24h", "to": "now"},
-    "templating": {"list": [{
-        "name": "pool",
-        "label": "Pool",
-        "type": "query",
-        "datasource": DS,
-        "query": {"qryType": 1, "query": "label_values(zfs_pool_health_code, pool)",
-                  "refId": "PrometheusVariableQueryEditor-VariableQuery"},
-        "refresh": 1,
-        "includeAll": True,
-        "allValue": ".*",
-        "multi": True,
-        "current": {"text": ["All"], "value": ["$__all"]},
-        "options": [],
-        "sort": 1,
-    }]},
+    "templating": {"list": [
+        # One machine at a time: not multi-select and no "All". See the comment on HOST
+        # above for why two hosts at once would silently merge same-named pools.
+        query_var("host", "label_values(zfs_pool_health_code, host)",
+                  label="Host", multi=False, include_all=False),
+        # Pools are listed for the selected host only, so switching hosts re-populates
+        # the picker rather than offering pools that do not exist on this machine.
+        query_var("pool", 'label_values(zfs_pool_health_code{host=~"$host"}, pool)',
+                  label="Pool"),
+    ]},
     "annotations": {"list": [{
         "builtIn": 1, "datasource": {"type": "datasource", "uid": "grafana"},
         "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)",
diff --git a/dashboards/smart.json b/dashboards/smart.json
index ed48cd1..f6c9867 100644
--- a/dashboards/smart.json
+++ b/dashboards/smart.json
@@ -19,6 +19,27 @@
   },
   "templating": {
     "list": [
+      {
+        "name": "host",
+        "label": "Host",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "query": {
+          "qryType": 1,
+          "query": "label_values(smartctl_device_smart_status, host)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": [],
+        "sort": 1,
+        "hide": 0,
+        "regex": ""
+      },
       {
         "name": "ata_device",
         "label": "SATA drive",
@@ -29,7 +50,7 @@
         },
         "query": {
           "qryType": 1,
-          "query": "label_values(smartctl:row{bus=\"ata\"}, row)",
+          "query": "label_values(smartctl:row{bus=\"ata\", host=~\"$host\"}, row)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
@@ -59,7 +80,7 @@
         },
         "query": {
           "qryType": 1,
-          "query": "label_values(smartctl:row{bus=\"sas\"}, row)",
+          "query": "label_values(smartctl:row{bus=\"sas\", host=~\"$host\"}, row)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
@@ -166,7 +187,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smartctl:health_code == 0) or vector(0)",
+          "expr": "count(smartctl:health_code{host=~\"$host\"} == 0) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -233,7 +254,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smartctl:health_code == 1) or vector(0)",
+          "expr": "count(smartctl:health_code{host=~\"$host\"} == 1) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -300,7 +321,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smartctl:health_code == 2) or vector(0)",
+          "expr": "count(smartctl:health_code{host=~\"$host\"} == 2) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -363,7 +384,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smartctl_device_smart_status)",
+          "expr": "count(smartctl_device_smart_status{host=~\"$host\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -435,7 +456,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max(smartctl_device_temperature{temperature_type=\"current\"})",
+          "expr": "max(smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -499,7 +520,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)))",
+          "expr": "max(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)))",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -664,7 +685,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl:health_code * on(device) group_left(model_name, serial_number, protocol) smartctl_device",
+          "expr": "smartctl:health_code{host=~\"$host\"} * on(device) group_left(model_name, serial_number, protocol) smartctl_device{host=~\"$host\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -676,7 +697,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_temperature{temperature_type=\"current\"})",
+          "expr": "max by (device) (smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"})",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -688,7 +709,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600))",
+          "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600))",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -700,7 +721,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_capacity_bytes)",
+          "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\"})",
           "refId": "D",
           "instant": true,
           "range": false,
@@ -841,7 +862,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_temperature{temperature_type=\"current\"}",
+          "expr": "smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -948,7 +969,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl:health_code{device=~\"$ata_device\"}",
+          "expr": "smartctl:health_code{host=~\"$host\", device=~\"$ata_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1020,7 +1041,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_temperature{device=~\"$ata_device\", temperature_type=\"current\"}",
+          "expr": "smartctl_device_temperature{host=~\"$host\", device=~\"$ata_device\", temperature_type=\"current\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1084,7 +1105,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$ata_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$ata_device\"} * 3600))",
+          "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\", host=~\"$host\", device=~\"$ata_device\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\", host=~\"$host\", device=~\"$ata_device\"} * 3600))",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1148,7 +1169,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_capacity_bytes{device=~\"$ata_device\"}",
+          "expr": "smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$ata_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1211,7 +1232,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_power_cycle_count{device=~\"$ata_device\"}",
+          "expr": "smartctl_device_power_cycle_count{host=~\"$host\", device=~\"$ata_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1278,7 +1299,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"5\", attribute_value_type=\"raw\"}",
+          "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_id=\"5\", attribute_value_type=\"raw\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1345,7 +1366,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"197\", attribute_value_type=\"raw\"}",
+          "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_id=\"197\", attribute_value_type=\"raw\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1442,7 +1463,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device{device=~\"$ata_device\"}",
+          "expr": "smartctl_device{host=~\"$host\", device=~\"$ata_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1454,7 +1475,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$ata_device\"})",
+          "expr": "max by (device) (smartctl_device_rotation_rate{host=~\"$host\", device=~\"$ata_device\"})",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -1466,7 +1487,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$ata_device\"})",
+          "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$ata_device\"})",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -1623,7 +1644,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"}",
+          "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"value\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1635,7 +1656,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"worst\"})",
+          "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"worst\"})",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -1647,7 +1668,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})",
+          "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"thresh\"})",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -1659,7 +1680,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"raw\"})",
+          "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"raw\"})",
           "refId": "D",
           "instant": true,
           "range": false,
@@ -1671,7 +1692,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"} - on(device, attribute_id) smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})",
+          "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"value\"} - on(device, attribute_id) smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"thresh\"})",
           "refId": "E",
           "instant": true,
           "range": false,
@@ -1831,7 +1852,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl:health_code{device=~\"$sas_device\"}",
+          "expr": "smartctl:health_code{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1903,7 +1924,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_temperature{device=~\"$sas_device\", temperature_type=\"current\"}",
+          "expr": "smartctl_device_temperature{host=~\"$host\", device=~\"$sas_device\", temperature_type=\"current\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1967,7 +1988,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$sas_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$sas_device\"} * 3600))",
+          "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\", host=~\"$host\", device=~\"$sas_device\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\", host=~\"$host\", device=~\"$sas_device\"} * 3600))",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2031,7 +2052,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_capacity_bytes{device=~\"$sas_device\"}",
+          "expr": "smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2095,7 +2116,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_interface_speed{device=~\"$sas_device\", speed_type=\"current\"} * 1000",
+          "expr": "smartctl_device_interface_speed{host=~\"$host\", device=~\"$sas_device\", speed_type=\"current\"} * 1000",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2162,7 +2183,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}",
+          "expr": "smartctl_scsi_grown_defect_list{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2229,7 +2250,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"} + smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}",
+          "expr": "smartctl_read_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"} + smartctl_write_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2326,7 +2347,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device{device=~\"$sas_device\"}",
+          "expr": "smartctl_device{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2338,7 +2359,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$sas_device\"})",
+          "expr": "max by (device) (smartctl_device_rotation_rate{host=~\"$host\", device=~\"$sas_device\"})",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -2350,7 +2371,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$sas_device\"})",
+          "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$sas_device\"})",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -2485,7 +2506,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_read_errors_corrected_by_eccfast{device=~\"$sas_device\"}",
+          "expr": "smartctl_read_errors_corrected_by_eccfast{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -2497,7 +2518,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_read_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}",
+          "expr": "smartctl_read_errors_corrected_by_eccdelayed{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "B",
           "instant": false,
           "range": true,
@@ -2509,7 +2530,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_read_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}",
+          "expr": "smartctl_read_errors_corrected_by_rereads_rewrites{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "C",
           "instant": false,
           "range": true,
@@ -2521,7 +2542,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_write_errors_corrected_by_eccfast{device=~\"$sas_device\"}",
+          "expr": "smartctl_write_errors_corrected_by_eccfast{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "D",
           "instant": false,
           "range": true,
@@ -2533,7 +2554,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_write_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}",
+          "expr": "smartctl_write_errors_corrected_by_eccdelayed{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "E",
           "instant": false,
           "range": true,
@@ -2545,7 +2566,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_write_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}",
+          "expr": "smartctl_write_errors_corrected_by_rereads_rewrites{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "F",
           "instant": false,
           "range": true,
@@ -2631,7 +2652,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}",
+          "expr": "smartctl_scsi_grown_defect_list{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -2643,7 +2664,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"}",
+          "expr": "smartctl_read_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "B",
           "instant": false,
           "range": true,
@@ -2655,7 +2676,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}",
+          "expr": "smartctl_write_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "C",
           "instant": false,
           "range": true,
@@ -2735,7 +2756,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smartctl_device_smart_status) - count(count by (device) (smart_selftest_last_lifetime_hours)) or count(smartctl_device_smart_status)",
+          "expr": "count(smartctl_device_smart_status{host=~\"$host\"}) - count(count by (device) (smart_selftest_last_lifetime_hours{host=~\"$host\"})) or count(smartctl_device_smart_status{host=~\"$host\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2802,7 +2823,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smart_selftest_last_passed == 0) or vector(0)",
+          "expr": "count(smart_selftest_last_passed{host=~\"$host\"} == 0) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2875,7 +2896,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{test_type=\"extended\"})",
+          "expr": "max(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{host=~\"$host\", test_type=\"extended\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2979,7 +3000,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "label_join(smart_selftest_last_passed, \"drive_test\", \"/\", \"device\", \"test_type\")",
+          "expr": "label_join(smart_selftest_last_passed{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\")",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2991,7 +3012,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (drive_test) (label_join(smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))",
+          "expr": "max by (drive_test) (label_join(smart_selftest_last_lifetime_hours{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -3003,7 +3024,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (drive_test) (label_join(smart_selftest_count, \"drive_test\", \"/\", \"device\", \"test_type\"))",
+          "expr": "max by (drive_test) (label_join(smart_selftest_count{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -3015,7 +3036,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (drive_test) (label_join(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))",
+          "expr": "max by (drive_test) (label_join(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))",
           "refId": "D",
           "instant": true,
           "range": false,
@@ -3166,7 +3187,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_attribute{attribute_id=\"5\", attribute_value_type=\"raw\"}",
+              "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=\"5\", attribute_value_type=\"raw\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3252,7 +3273,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_attribute{attribute_id=~\"197|198\", attribute_value_type=\"raw\"}",
+              "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=~\"197|198\", attribute_value_type=\"raw\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3338,7 +3359,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_attribute{attribute_id=\"199\", attribute_value_type=\"raw\"}",
+              "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=\"199\", attribute_value_type=\"raw\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3425,7 +3446,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_bytes_written",
+              "expr": "smartctl_device_bytes_written{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3437,7 +3458,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_bytes_read",
+              "expr": "smartctl_device_bytes_read{host=~\"$host\"}",
               "refId": "B",
               "instant": false,
               "range": true,
@@ -3545,7 +3566,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl:health_code",
+              "expr": "smartctl:health_code{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3645,7 +3666,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_smartctl_exit_status",
+              "expr": "smartctl_device_smartctl_exit_status{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3732,7 +3753,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smart_selftest_collection_duration_seconds",
+              "expr": "smart_selftest_collection_duration_seconds{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3818,7 +3839,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_devices",
+              "expr": "smartctl_devices{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3830,7 +3851,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smart_selftest_devices",
+              "expr": "smart_selftest_devices{host=~\"$host\"}",
               "refId": "B",
               "instant": false,
               "range": true,
diff --git a/dashboards/zfs-pools.json b/dashboards/zfs-pools.json
index 0721bb8..c36a42a 100644
--- a/dashboards/zfs-pools.json
+++ b/dashboards/zfs-pools.json
@@ -18,6 +18,27 @@
   },
   "templating": {
     "list": [
+      {
+        "name": "host",
+        "label": "Host",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "query": {
+          "qryType": 1,
+          "query": "label_values(zfs_pool_health_code, host)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": [],
+        "sort": 1,
+        "hide": 0,
+        "regex": ""
+      },
       {
         "name": "pool",
         "label": "Pool",
@@ -28,13 +49,17 @@
         },
         "query": {
           "qryType": 1,
-          "query": "label_values(zfs_pool_health_code, pool)",
+          "query": "label_values(zfs_pool_health_code{host=~\"$host\"}, pool)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
         "includeAll": true,
-        "allValue": ".*",
         "multi": true,
+        "options": [],
+        "sort": 1,
+        "hide": 0,
+        "regex": "",
+        "allValue": ".*",
         "current": {
           "text": [
             "All"
@@ -42,9 +67,7 @@
           "value": [
             "$__all"
           ]
-        },
-        "options": [],
-        "sort": 1
+        }
       }
     ]
   },
@@ -184,7 +207,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_health_code{pool=~\"$pool\"}",
+          "expr": "zfs_pool_health_code{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -251,7 +274,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(zfs_pool_data_errors{pool=~\"$pool\"}) or vector(0)",
+          "expr": "sum(zfs_pool_data_errors{host=~\"$host\", pool=~\"$pool\"}) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -318,7 +341,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(zfs_pool_vdev_errors_total{pool=~\"$pool\"}) or vector(0)",
+          "expr": "sum(zfs_pool_vdev_errors_total{host=~\"$host\", pool=~\"$pool\"}) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -390,7 +413,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "time() - max(zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"})",
+          "expr": "time() - max(zfs_pool_last_scrub_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -469,7 +492,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max(zfs_pool_scrub_in_progress{pool=~\"$pool\"})",
+          "expr": "max(zfs_pool_scrub_in_progress{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -481,7 +504,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max(zfs_pool_resilver_in_progress{pool=~\"$pool\"})",
+          "expr": "max(zfs_pool_resilver_in_progress{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -548,7 +571,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(zfs_dataset_encrypted{pool=~\"$pool\"} == 1 and zfs_dataset_key_available{pool=~\"$pool\"} == 0) or vector(0)",
+          "expr": "count(zfs_dataset_encrypted{host=~\"$host\", pool=~\"$pool\"} == 1 and zfs_dataset_key_available{host=~\"$host\", pool=~\"$pool\"} == 0) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -618,7 +641,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_capacity_ratio{pool=~\"$pool\"}",
+          "expr": "zfs_pool_capacity_ratio{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -688,7 +711,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}",
+          "expr": "zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -752,7 +775,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}",
+          "expr": "zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -819,7 +842,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count((time() - zfs_snapshot_newest_timestamp_seconds{pool=~\"$pool\"}) > on(label) group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)",
+          "expr": "count((time() - zfs_snapshot_newest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"}) > on(label) group_left() (2 * zfs_snapshot_interval_seconds{host=~\"$host\"})) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -883,7 +906,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})",
+          "expr": "time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -970,7 +993,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_allocated_bytes{pool=~\"$pool\"}",
+          "expr": "zfs_pool_allocated_bytes{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -982,7 +1005,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}",
+          "expr": "zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "B",
           "instant": false,
           "range": true,
@@ -1070,7 +1093,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}",
+          "expr": "zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -1206,7 +1229,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}",
+          "expr": "zfs_pool_size_bytes{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1218,7 +1241,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (pool) (zfs_pool_allocated_bytes{pool=~\"$pool\"})",
+          "expr": "max by (pool) (zfs_pool_allocated_bytes{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -1230,7 +1253,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (pool) (zfs_pool_free_bytes{pool=~\"$pool\"})",
+          "expr": "max by (pool) (zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -1242,7 +1265,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (pool) (zfs_pool_capacity_ratio{pool=~\"$pool\"})",
+          "expr": "max by (pool) (zfs_pool_capacity_ratio{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "D",
           "instant": true,
           "range": false,
@@ -1254,7 +1277,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (pool) (zfs_pool_fragmentation_ratio{pool=~\"$pool\"})",
+          "expr": "max by (pool) (zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "E",
           "instant": true,
           "range": false,
@@ -1266,7 +1289,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (pool) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})",
+          "expr": "max by (pool) (zfs_pool_snapshot_used_bytes{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "F",
           "instant": true,
           "range": false,
@@ -1278,7 +1301,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (pool) (zfs_pool_dedup_ratio{pool=~\"$pool\"})",
+          "expr": "max by (pool) (zfs_pool_dedup_ratio{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "G",
           "instant": true,
           "range": false,
@@ -1481,7 +1504,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_vdev_state_code{pool=~\"$pool\"}",
+          "expr": "zfs_pool_vdev_state_code{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1589,7 +1612,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_vdev_errors_total{pool=~\"$pool\"}",
+          "expr": "zfs_pool_vdev_errors_total{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -1737,7 +1760,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"} * 1000",
+          "expr": "zfs_pool_last_scrub_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"} * 1000",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1749,7 +1772,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_last_scrub_duration_seconds{pool=~\"$pool\"}",
+          "expr": "zfs_pool_last_scrub_duration_seconds{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -1761,7 +1784,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_last_scrub_errors{pool=~\"$pool\"}",
+          "expr": "zfs_pool_last_scrub_errors{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -1773,7 +1796,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_last_scrub_repaired_bytes{pool=~\"$pool\"}",
+          "expr": "zfs_pool_last_scrub_repaired_bytes{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "D",
           "instant": true,
           "range": false,
@@ -1785,7 +1808,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_last_resilver_end_timestamp_seconds{pool=~\"$pool\"} * 1000",
+          "expr": "zfs_pool_last_resilver_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"} * 1000",
           "refId": "E",
           "instant": true,
           "range": false,
@@ -1903,7 +1926,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_pool_data_errors{pool=~\"$pool\"}",
+          "expr": "zfs_pool_data_errors{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -1987,7 +2010,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{pool=~\"$pool\"})",
+          "expr": "time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2059,7 +2082,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "min by (label) (zfs_snapshot_count{pool=~\"$pool\"} / on(label) group_left() zfs_snapshot_retention_target)",
+          "expr": "min by (label) (zfs_snapshot_count{host=~\"$host\", pool=~\"$pool\"} / on(label) group_left() zfs_snapshot_retention_target{host=~\"$host\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2122,7 +2145,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "time() - min by (label) (zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})",
+          "expr": "time() - min by (label) (zfs_pool_snapshot_oldest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2208,7 +2231,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by (label) (zfs_snapshot_count{pool=~\"$pool\"})",
+          "expr": "sum by (label) (zfs_snapshot_count{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -2295,7 +2318,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by (label) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})",
+          "expr": "sum by (label) (zfs_pool_snapshot_used_bytes{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -2362,7 +2385,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(node_systemd_unit_state{name=~\"zfs-(snapshot|scrub).*\",state=\"failed\"}) or vector(0)",
+          "expr": "sum(node_systemd_unit_state{host=~\"$host\", name=~\"zfs-(snapshot|scrub).*\", state=\"failed\"}) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2425,7 +2448,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "time() - node_systemd_timer_last_trigger_seconds{name=~\"zfs-.*\"}",
+          "expr": "time() - node_systemd_timer_last_trigger_seconds{host=~\"$host\", name=~\"zfs-.*\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2525,7 +2548,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}",
+          "expr": "zfs_dataset_used_bytes{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -2630,7 +2653,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}",
+          "expr": "zfs_dataset_used_bytes{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2642,7 +2665,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (dataset) (zfs_dataset_used_by_dataset_bytes{pool=~\"$pool\"})",
+          "expr": "max by (dataset) (zfs_dataset_used_by_dataset_bytes{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -2654,7 +2677,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (dataset) (zfs_dataset_used_by_snapshots_bytes{pool=~\"$pool\"})",
+          "expr": "max by (dataset) (zfs_dataset_used_by_snapshots_bytes{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -2666,7 +2689,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (dataset) (zfs_dataset_referenced_bytes{pool=~\"$pool\"})",
+          "expr": "max by (dataset) (zfs_dataset_referenced_bytes{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "D",
           "instant": true,
           "range": false,
@@ -2678,7 +2701,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (dataset) (zfs_dataset_logical_used_bytes{pool=~\"$pool\"})",
+          "expr": "max by (dataset) (zfs_dataset_logical_used_bytes{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "E",
           "instant": true,
           "range": false,
@@ -2690,7 +2713,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (dataset) (zfs_dataset_available_bytes{pool=~\"$pool\"})",
+          "expr": "max by (dataset) (zfs_dataset_available_bytes{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "F",
           "instant": true,
           "range": false,
@@ -2702,7 +2725,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (dataset) (zfs_dataset_compression_ratio{pool=~\"$pool\"})",
+          "expr": "max by (dataset) (zfs_dataset_compression_ratio{host=~\"$host\", pool=~\"$pool\"})",
           "refId": "G",
           "instant": true,
           "range": false,
@@ -2714,7 +2737,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (dataset) (zfs_dataset_quota_bytes{pool=~\"$pool\"} > 0)",
+          "expr": "max by (dataset) (zfs_dataset_quota_bytes{host=~\"$host\", pool=~\"$pool\"} > 0)",
           "refId": "H",
           "instant": true,
           "range": false,
@@ -2841,7 +2864,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_dataset_compression_ratio{pool=~\"$pool\"} * on(pool, dataset) group_left(compression) zfs_dataset_compression{pool=~\"$pool\"}",
+          "expr": "zfs_dataset_compression_ratio{host=~\"$host\", pool=~\"$pool\"} * on(pool, dataset) group_left(compression) zfs_dataset_compression{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2991,7 +3014,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "zfs_dataset_keystatus_code{pool=~\"$pool\"} * on(pool, dataset) group_left(encryption) zfs_dataset_encryption{pool=~\"$pool\"}",
+          "expr": "zfs_dataset_keystatus_code{host=~\"$host\", pool=~\"$pool\"} * on(pool, dataset) group_left(encryption) zfs_dataset_encryption{host=~\"$host\", pool=~\"$pool\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -3118,7 +3141,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "rate(node_zfs_zpool_dataset_nread{zpool=~\"$pool\"}[$__rate_interval])",
+          "expr": "rate(node_zfs_zpool_dataset_nread{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -3130,7 +3153,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "-rate(node_zfs_zpool_dataset_nwritten{zpool=~\"$pool\"}[$__rate_interval])",
+          "expr": "-rate(node_zfs_zpool_dataset_nwritten{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])",
           "refId": "B",
           "instant": false,
           "range": true,
@@ -3216,7 +3239,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "rate(node_zfs_zpool_dataset_reads{zpool=~\"$pool\"}[$__rate_interval])",
+          "expr": "rate(node_zfs_zpool_dataset_reads{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -3228,7 +3251,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "-rate(node_zfs_zpool_dataset_writes{zpool=~\"$pool\"}[$__rate_interval])",
+          "expr": "-rate(node_zfs_zpool_dataset_writes{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])",
           "refId": "B",
           "instant": false,
           "range": true,
@@ -3315,7 +3338,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "node_zfs_arc_size",
+          "expr": "node_zfs_arc_size{host=~\"$host\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -3327,7 +3350,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "node_zfs_arc_c_max",
+          "expr": "node_zfs_arc_c_max{host=~\"$host\"}",
           "refId": "B",
           "instant": false,
           "range": true,
@@ -3339,7 +3362,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "node_zfs_arc_c_min",
+          "expr": "node_zfs_arc_c_min{host=~\"$host\"}",
           "refId": "C",
           "instant": false,
           "range": true,
@@ -3426,7 +3449,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min(rate(node_zfs_arc_hits[$__rate_interval]) + rate(node_zfs_arc_misses[$__rate_interval]), 1)",
+          "expr": "rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval]) / clamp_min(rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval]) + rate(node_zfs_arc_misses{host=~\"$host\"}[$__rate_interval]), 1)",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -3513,7 +3536,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "rate(node_zfs_arc_hits[$__rate_interval])",
+          "expr": "rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval])",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -3525,7 +3548,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "rate(node_zfs_arc_misses[$__rate_interval])",
+          "expr": "rate(node_zfs_arc_misses{host=~\"$host\"}[$__rate_interval])",
           "refId": "B",
           "instant": false,
           "range": true,
@@ -3624,7 +3647,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "zfs_pool_freeing_bytes{pool=~\"$pool\"}",
+              "expr": "zfs_pool_freeing_bytes{host=~\"$host\", pool=~\"$pool\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3636,7 +3659,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "zfs_pool_leaked_bytes{pool=~\"$pool\"}",
+              "expr": "zfs_pool_leaked_bytes{host=~\"$host\", pool=~\"$pool\"}",
               "refId": "B",
               "instant": false,
               "range": true,
@@ -3648,7 +3671,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "zfs_pool_checkpoint_bytes{pool=~\"$pool\"}",
+              "expr": "zfs_pool_checkpoint_bytes{host=~\"$host\", pool=~\"$pool\"}",
               "refId": "C",
               "instant": false,
               "range": true,
@@ -3734,7 +3757,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "zfs_pool_dedup_ratio{pool=~\"$pool\"}",
+              "expr": "zfs_pool_dedup_ratio{host=~\"$host\", pool=~\"$pool\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3872,7 +3895,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "zfs_pool_health_code{pool=~\"$pool\"}",
+              "expr": "zfs_pool_health_code{host=~\"$host\", pool=~\"$pool\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -4005,7 +4028,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "zfs_dataset_recordsize_bytes{pool=~\"$pool\"}",
+              "expr": "zfs_dataset_recordsize_bytes{host=~\"$host\", pool=~\"$pool\"}",
               "refId": "A",
               "instant": true,
               "range": false,
@@ -4017,7 +4040,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "zfs_dataset_quota_bytes{pool=~\"$pool\"}",
+              "expr": "zfs_dataset_quota_bytes{host=~\"$host\", pool=~\"$pool\"}",
               "refId": "B",
               "instant": true,
               "range": false,
@@ -4029,7 +4052,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "zfs_dataset_refquota_bytes{pool=~\"$pool\"}",
+              "expr": "zfs_dataset_refquota_bytes{host=~\"$host\", pool=~\"$pool\"}",
               "refId": "C",
               "instant": true,
               "range": false,
@@ -4041,7 +4064,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "zfs_dataset_autosnapshot_enabled{pool=~\"$pool\"}",
+              "expr": "zfs_dataset_autosnapshot_enabled{host=~\"$host\", pool=~\"$pool\"}",
               "refId": "D",
               "instant": true,
               "range": false,
@@ -4173,7 +4196,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "zfs_metrics_collection_duration_seconds",
+              "expr": "zfs_metrics_collection_duration_seconds{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -4260,7 +4283,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "time() - node_textfile_mtime_seconds",
+              "expr": "time() - node_textfile_mtime_seconds{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -4358,7 +4381,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "node_textfile_scrape_error",
+              "expr": "node_textfile_scrape_error{host=~\"$host\"}",
               "refId": "B",
               "instant": false,
               "range": true,
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 0000000..c508296
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,27 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1787736819,
+        "narHash": "sha256-cV5xEJJK3BvhU8rEd4mC9UsmDi5qscv/kzGPhBRC5WA=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "9fbb54b33e91ee4ca368e35a78e0613c720600b3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 0000000..1a9c661
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,109 @@
+{
+  description = "Host telemetry collectors and a Prometheus/Loki/Grafana server, as two separable NixOS modules";
+
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+
+  outputs = { self, nixpkgs }:
+    let
+      systems = [ "x86_64-linux" "aarch64-linux" ];
+      forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system});
+    in
+    {
+      # Paths rather than functions, so the modules are evaluated with the *consumer's*
+      # nixpkgs. This flake's own nixpkgs input therefore only affects packages, checks
+      # and devShells -- importing these modules never pulls in a second nixpkgs.
+      nixosModules = {
+        collectors = ./modules/collectors.nix;
+        server = ./modules/server.nix;
+        default = ./modules;
+      };
+
+      # The textfile collectors, runnable by hand. Both take --output PATH and write
+      # Prometheus exposition format; running them against a scratch file is how you
+      # check what a machine actually reports.
+      packages = forAllSystems (pkgs:
+        # callPackages, not callPackage: the file returns an attrset of
+        # derivations, and callPackage would wrap the whole set in
+        # override/overrideDerivation attrs that are not derivations.
+        let collectors = pkgs.callPackages ./modules/packages.nix { };
+        in collectors // { default = collectors.zfs-metrics; }
+      );
+
+      checks = forAllSystems (pkgs: {
+        # promtool over the recording rules. The server module also runs this at build
+        # time via services.prometheus.checkConfig, but that only fires for people who
+        # actually deploy it.
+        prometheus-rules = pkgs.runCommand "check-prometheus-rules"
+          { nativeBuildInputs = [ pkgs.prometheus.cli ]; }
+          ''
+            promtool check rules ${./rules}/*.yml
+            touch "$out"
+          '';
+
+        # The dashboard JSON is generated by the scripts next to it and committed, so it
+        # can rot silently if someone edits a generator without re-running it. Regenerate
+        # here and diff.
+        dashboards-current = pkgs.runCommand "check-dashboards-current"
+          { nativeBuildInputs = [ pkgs.python3 pkgs.diffutils ]; }
+          ''
+            cp -r ${./dashboards} work
+            chmod -R u+w work
+            cd work
+
+            # Note the loop variable is not called "out": that is the derivation's own
+            # output path, and shadowing it makes the final touch silently write
+            # somewhere else and the build fail with "failed to produce output path".
+            check() {
+              local generator="$1" committed="$2"
+              python3 "$generator" > regenerated.json
+              if ! diff -u "$committed" regenerated.json; then
+                echo
+                echo "dashboards/$committed is stale."
+                echo "Re-run: python3 dashboards/$generator > dashboards/$committed"
+                exit 1
+              fi
+            }
+
+            check gen_smart.py smart.json
+            check gen_zfs.py zfs-pools.json
+            touch "$out"
+          '';
+
+        # Both halves have to evaluate, separately and together. Catches option typos
+        # and missing mkIf guards without needing a real machine.
+        modules-eval = (nixpkgs.lib.nixosSystem {
+          system = pkgs.stdenv.hostPlatform.system;
+          modules = [
+            self.nixosModules.default
+            ({ lib, ... }: {
+              services.telemetry.collectors.enable = true;
+              services.telemetry.server = {
+                enable = true;
+                collectorHosts = [ "one" "two" ];
+                proxy.enable = true;
+                # Never read during evaluation; only has to satisfy the assertion.
+                grafana.secretKeyFile = "/run/secrets/grafana-secret-key";
+              };
+              boot.loader.grub.devices = [ "/dev/sda" ];
+              fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; };
+              system.stateVersion = "25.11";
+            })
+          ];
+        }).config.system.build.toplevel;
+      });
+
+      devShells = forAllSystems (pkgs: {
+        default = pkgs.mkShell {
+          packages = [
+            pkgs.python3 # the dashboard generators and collectors
+            pkgs.prometheus.cli # promtool, for the recording rules
+            pkgs.prometheus # a real Prometheus to run dashboard queries against
+            pkgs.jq
+            pkgs.smartmontools
+          ];
+        };
+      });
+
+      formatter = forAllSystems (pkgs: pkgs.nixpkgs-fmt);
+    };
+}
diff --git a/lgtm.nix b/lgtm.nix
deleted file mode 100644
index a4144ec..0000000
--- a/lgtm.nix
+++ /dev/null
@@ -1,534 +0,0 @@
-{ pkgs, config, lib, ... }:
-
-let
-	# The hostname the container uses to reach the host's exporters.
-	hostName = "poweredge";
-
-	# One scrape interval to rule them all. This MUST match the Grafana Prometheus
-	# datasource's `timeInterval` below, otherwise Grafana computes $__rate_interval
-	# from the wrong assumption and every rate()-based panel intermittently renders
-	# "No data". (Prometheus' own default is 1m, which is what broke the Node Exporter
-	# Full dashboard: $__rate_interval resolved to 60s, so rate(x[60s]) over 60s-spaced
-	# samples usually saw fewer than the two samples rate() requires.)
-	scrapeInterval = "15s";
-
-	# Where the host's zpool collector drops its metrics for node_exporter to pick up.
-	textfileDir = "/var/lib/prometheus-node-exporter-textfile";
-
-	ports = {
-		node     = 9100;
-		smartctl = 9633;
-		loki     = 3100;
-		prometheus = 9090;
-		grafana  = 3000;
-	};
-
-	# Stable datasource UIDs, referenced by the dashboards below.
-	dsUid = { prometheus = "prometheus"; loki = "loki"; };
-
-	##########################################################################
-	# Dashboard plumbing
-	##########################################################################
-
-	# Dashboards exported from grafana.com carry an `__inputs` block and reference
-	# their datasource as "${DS_PROMETHEUS}". That placeholder is only substituted by
-	# Grafana's interactive *import* flow — file-based provisioning never runs it, so
-	# the uid stays literal and every panel fails to resolve a datasource. This filter
-	# performs the substitution at build time, using our fixed datasource UIDs.
-	dashboardFixup = pkgs.writeText "grafana-dashboard-fixup.jq" ''
-		def dsuid($pluginId):
-			if   $pluginId == "prometheus" then $prom
-			elif $pluginId == "loki"       then $loki
-			else "" end;
-
-		def subst($name; $uid):
-			walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end);
-
-		reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId)))
-		| del(.__inputs, .__requires)
-		# Provisioned dashboards must not carry an id, or Grafana rejects them on
-		# collision with whatever already occupies that id in its database.
-		| .id = null
-	'';
-
-	fixupDashboard = name: src: pkgs.runCommand "grafana-dashboard-${name}.json"
-		{ nativeBuildInputs = [ pkgs.jq ]; }
-		''
-			jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \
-			   --arg loki ${lib.escapeShellArg dsUid.loki} \
-			   -f ${dashboardFixup} < ${src} > "$out"
-		'';
-
-	# grafana.com dashboard, pinned by revision.
-	mkGrafanaComDashboard = { name, id, rev, sha256 }:
-		fixupDashboard name (pkgs.fetchurl {
-			url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download";
-			inherit sha256;
-		});
-
-	dashboards = {
-		# https://grafana.com/grafana/dashboards/1860 — Node Exporter Full
-		node-exporter-full = mkGrafanaComDashboard {
-			name = "node-exporter-full"; id = 1860; rev = 45;
-			sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY=";
-		};
-
-		# https://grafana.com/grafana/dashboards/22604 — Smartctl Exporter
-		smartctl-exporter = mkGrafanaComDashboard {
-			name = "smartctl-exporter"; id = 22604; rev = 2;
-			sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88=";
-		};
-
-		# Dashboard 15362 ("ZFS Pool Metrics") used to live here. It is an *InfluxDB*
-		# dashboard — its queries are InfluxQL against a `zpool_stats` measurement — so
-		# it can never render against Prometheus, no matter how it is provisioned. This
-		# is a Prometheus-native replacement built on the metrics we actually export:
-		# node_exporter's zfs collector plus the zpool textfile collector below.
-		zfs-pools = fixupDashboard "zfs-pools" ./dashboards/zfs-pools.json;
-
-		# CrystalDiskInfo-style per-drive SMART view. Generated by
-		# ./dashboards/gen_smart.py; the Good/Caution/Bad verdict it displays comes
-		# from the recording rules in ./rules/smartctl.yml.
-		smart = fixupDashboard "smart" ./dashboards/smart.json;
-	};
-
-	##########################################################################
-	# ZFS metrics (host)
-	##########################################################################
-
-	# node_exporter's zfs collector reads /proc/spl/kstat/zfs, which gives us ARC
-	# statistics, per-dataset objset counters and pool state -- but nothing about
-	# capacity, fragmentation, vdev errors, scrub history, compression, encryption or
-	# snapshots. OpenZFS 2.2+ also no longer publishes the per-pool `io` kstat, so
-	# node_zfs_zpool_* is largely empty. Everything else comes from the zpool/zfs CLIs
-	# via ./collectors/zfs-metrics.py, scraped through the textfile collector.
-	#
-	# The wrapper exists only to put the matching zfs userland on PATH; the collector
-	# itself is a plain script so it can be run and tested by hand.
-	snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ];
-
-	retentionArgs = lib.optionals config.services.zfs.autoSnapshot.enable (
-		lib.concatMap (label: [
-			"--retention"
-			"${label}=${toString config.services.zfs.autoSnapshot.${label}}"
-		]) snapshotLabels
-	);
-
-	zfsCollector = pkgs.writeShellApplication {
-		name = "zfs-textfile-collector";
-		runtimeInputs = [ config.boot.zfs.package ];
-		text = ''
-			exec ${pkgs.python3}/bin/python3 ${./collectors/zfs-metrics.py} \
-				--output ${textfileDir}/zfs.prom ${lib.escapeShellArgs retentionArgs}
-		'';
-	};
-
-	# smartctl_exporter reads only the entry *count* out of each drive's self-test log
-	# and discards the log table, so it cannot say when a drive last ran a short or
-	# extended test, or whether it passed. This parses `smartctl -j -l selftest`
-	# directly. Device labels are bare names ("sda") so they join against the
-	# exporter's own series.
-	smartSelftestCollector = pkgs.writeShellApplication {
-		name = "smart-selftest-collector";
-		runtimeInputs = [ pkgs.smartmontools ];
-		text = ''
-			exec ${pkgs.python3}/bin/python3 ${./collectors/smart-selftest.py} \
-				--output ${textfileDir}/smart-selftest.prom
-		'';
-	};
-in
-{
-	##########################################################################
-	# Host: log shipping
-	##########################################################################
-
-	services.alloy.enable = true;
-
-	environment.etc."alloy/config.alloy".text = ''
-		loki.source.journal "journal" {
-			forward_to    = [loki.write.default.receiver]
-			relabel_rules = loki.relabel.journal.rules
-			max_age       = "12h"
-		}
-
-		loki.relabel "journal" {
-			forward_to = []
-			rule {
-				source_labels = ["__journal__systemd_unit"]
-				target_label  = "unit"
-			}
-			rule {
-				source_labels = ["__journal__hostname"]
-				target_label  = "host"
-			}
-			rule {
-				source_labels = ["__journal_priority_keyword"]
-				target_label  = "level"
-			}
-		}
-
-		loki.write "default" {
-			endpoint {
-				url = "http://lgtm:${toString ports.loki}/loki/api/v1/push"
-			}
-		}
-	'';
-
-	##########################################################################
-	# Host: telemetry sources
-	#
-	# These have to live on the host, not in the container: they read host
-	# hardware and kernel state (/proc, /sys, SMART, ZFS ioctls) that a
-	# container has no view of. Only the web services (Grafana, Prometheus,
-	# Loki) run in the container.
-	##########################################################################
-
-	services.prometheus.exporters.node = {
-		enable = true;
-		port = ports.node;
-		listenAddress = "0.0.0.0";
-
-		# NOTE: enabledCollectors *adds* to node_exporter's defaults — cpu, diskstats,
-		# filesystem, meminfo, zfs, hwmon, pressure and friends are already on, so
-		# listing them here did nothing. Only genuinely opt-in collectors belong here.
-		enabledCollectors = [
-			"systemd"    # node_systemd_* — the "Systemd" row
-			"processes"  # node_processes_* — "Processes Detailed States", "PIDs", "Threads"
-			# "interrupts"  # fills the "IRQ Detail" panel, but emits one series per
-			#               # (cpu, irq) pair — thousands of series on a many-core box.
-			#               # Enable only if you want that one panel badly enough.
-		];
-
-		extraFlags = [
-			"--collector.textfile.directory=${textfileDir}"
-			# node_systemd_unit_start_time_seconds, so the dashboards can show when the
-			# zfs-snapshot-*/zfs-scrub units last ran rather than only whether they failed.
-			"--collector.systemd.enable-start-time-metrics"
-		];
-	};
-
-	# Upstream hardens the unit with ProtectHome=true, which makes /home look empty and
-	# blanks out the filesystem panels for any dataset mounted under it. read-only still
-	# allows statfs while keeping the sandbox.
-	systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only";
-
-	services.prometheus.exporters.smartctl = {
-		enable = true;
-		port = ports.smartctl;
-		listenAddress = "0.0.0.0";
-		# Disks are direct-attached (AHCI/HBA), so auto-discovery via `smartctl --scan`
-		# finds them; no explicit `devices` list needed. Throttle how often we actually
-		# shell out to smartctl — SMART data changes slowly and polling spins up disks.
-		maxInterval = "60s";
-	};
-
-	# SMART self-test history. Reading the log does not spin an idle disk up, but the
-	# data only changes when a test finishes, so this runs far less often than the
-	# exporter scrapes.
-	systemd.services.smart-selftest-collector = {
-		description = "Collect SMART self-test history for the Prometheus node exporter";
-		serviceConfig = {
-			Type = "oneshot";
-			ExecStart = lib.getExe smartSelftestCollector;
-			User = "root";
-			StateDirectory = "prometheus-node-exporter-textfile";
-			StateDirectoryMode = "0755";
-			UMask = "0022";
-			TimeoutStartSec = "180s";
-
-			# Same shape as the smartctl exporter's own hardening: it needs raw device
-			# access to talk to the disks, and nothing else.
-			AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
-			CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
-			DevicePolicy = "closed";
-			DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ];
-			LockPersonality = true;
-			MemoryDenyWriteExecute = true;
-			NoNewPrivileges = true;
-			PrivateNetwork = true;
-			PrivateTmp = true;
-			ProtectControlGroups = true;
-			ProtectHome = true;
-			ProtectHostname = true;
-			ProtectKernelLogs = true;
-			ProtectKernelModules = true;
-			ProtectSystem = "strict";
-			RestrictAddressFamilies = [ "AF_UNIX" ];
-			RestrictNamespaces = true;
-			RestrictRealtime = true;
-			RestrictSUIDSGID = true;
-			SystemCallArchitectures = "native";
-		};
-	};
-
-	systemd.timers.smart-selftest-collector = {
-		description = "Collect SMART self-test history periodically";
-		wantedBy = [ "timers.target" ];
-		timerConfig = {
-			OnBootSec = "3min";
-			OnUnitActiveSec = "15min";
-			AccuracySec = "1min";
-			Unit = "smart-selftest-collector.service";
-		};
-	};
-
-	# Pool capacity and health, vdev errors, scrub history, dataset properties and
-	# snapshot retention. See zfsCollector above for why node_exporter's built-in zfs
-	# collector isn't enough on its own.
-	systemd.services.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled {
-		description = "Collect ZFS metrics for the Prometheus node exporter";
-		serviceConfig = {
-			Type = "oneshot";
-			ExecStart = lib.getExe zfsCollector;
-			User = "root";
-			StateDirectory = "prometheus-node-exporter-textfile";
-			StateDirectoryMode = "0755";
-			UMask = "0022";  # node_exporter runs as a different user and must read these
-			# Listing every snapshot on a busy pool is the slow part; keep this well
-			# above the subprocess timeout inside the collector.
-			TimeoutStartSec = "180s";
-
-			# Hardening — this only needs to read ZFS state and write one file.
-			# CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated
-			# on it; everything else is dropped.
-			CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ];
-			LockPersonality = true;
-			MemoryDenyWriteExecute = true;
-			NoNewPrivileges = true;
-			PrivateNetwork = true;
-			PrivateTmp = true;
-			ProtectControlGroups = true;
-			ProtectHome = true;
-			ProtectHostname = true;
-			ProtectKernelLogs = true;
-			ProtectKernelModules = true;
-			ProtectSystem = "strict";
-			RestrictAddressFamilies = [ "AF_UNIX" ];
-			RestrictNamespaces = true;
-			RestrictRealtime = true;
-			RestrictSUIDSGID = true;
-			SystemCallArchitectures = "native";
-			SystemCallFilter = [ "@system-service" ];
-		};
-	};
-
-	systemd.timers.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled {
-		description = "Collect ZFS metrics periodically";
-		wantedBy = [ "timers.target" ];
-		timerConfig = {
-			OnBootSec = "1min";
-			OnUnitActiveSec = "1min";
-			AccuracySec = "5s";
-			Unit = "zfs-textfile-collector.service";
-		};
-	};
-
-	# The collector used to write zpool.prom. Nothing removes a stale textfile once its
-	# writer is gone, and node_exporter would happily keep serving those frozen metrics
-	# forever, so delete it on activation.
-	systemd.tmpfiles.rules = lib.mkIf config.boot.zfs.enabled [
-		"r ${textfileDir}/zpool.prom"
-	];
-
-	# Reachable from the container, which sits on br-lan0 with its own address.
-	# (Port 9273 is gone — that was telegraf/zpool_influxdb, replaced by the textfile
-	# collector above, which needs no listener of its own.)
-	networking.firewall.allowedTCPPorts = [ ports.node ports.smartctl ];
-
-	##########################################################################
-	# Secrets
-	##########################################################################
-
-	sops.secrets.grafana-secret-key = {
-		sopsFile = ./resources/secrets/grafana.yaml;
-		key = "secret_key";
-	};
-
-	##########################################################################
-	# Container: Grafana / Prometheus / Loki
-	##########################################################################
-
-	containers.grafana = {
-		autoStart = true;
-		privateNetwork = true;
-		hostBridge = "br-lan0";
-		localMacAddress = "02:00:00:00:00:06";
-
-		bindMounts."/run/secrets/grafana-secret_key" = {
-			hostPath = config.sops.secrets.grafana-secret-key.path;
-			isReadOnly = true;
-		};
-
-		config = { lib, pkgs, config, ... }: {
-			# Network
-			networking.enableIPv6 = false;
-			networking.interfaces.eth0.useDHCP = true;
-			networking.firewall.allowedTCPPorts = [
-				80                 # Caddy -> Grafana
-				ports.loki         # host's Alloy pushes here
-				ports.prometheus   # handy for debugging scrape health directly
-			];
-
-			# ---------------------------------------------------------------
-			# Loki
-			# ---------------------------------------------------------------
-			services.loki = {
-				enable = true;
-				configuration = {
-					server = {
-						http_listen_port = ports.loki;
-						# Explicit, since the host's Alloy needs to reach it to push logs.
-						http_listen_address = "0.0.0.0";
-					};
-					auth_enabled = false;
-
-					common = {
-						path_prefix = "/var/lib/loki";
-						storage.filesystem = {
-							chunks_directory = "/var/lib/loki/chunks";
-							rules_directory = "/var/lib/loki/rules";
-						};
-						ring = {
-							instance_addr = "127.0.0.1";
-							kvstore.store = "inmemory";
-						};
-						replication_factor = 1;
-					};
-
-					schema_config.configs = [{
-						from = "2024-01-01";
-						store = "tsdb";
-						object_store = "filesystem";
-						schema = "v13";
-						index = { prefix = "index_"; period = "24h"; };
-					}];
-				};
-			};
-
-			# ---------------------------------------------------------------
-			# Prometheus
-			# ---------------------------------------------------------------
-			services.prometheus = {
-				enable = true;
-
-				# Without this Prometheus falls back to its built-in 1m scrape interval,
-				# which desynchronises from what Grafana assumes and breaks every
-				# rate()-based panel. See the comment on `scrapeInterval` at the top.
-				globalConfig = {
-					scrape_interval = scrapeInterval;
-					evaluation_interval = scrapeInterval;
-					scrape_timeout = "10s";
-				};
-
-				# Drive health verdict. Kept as recording rules rather than inlined in
-				# dashboard panels so the logic is written once and reviewable on its
-				# own. services.prometheus.checkConfig runs promtool at build time, so
-				# a broken rule fails the build instead of the service.
-				ruleFiles = [ ./rules/smartctl.yml ];
-
-				scrapeConfigs = [
-					{
-						job_name = "node";
-						static_configs = [{ targets = [ "${hostName}:${toString ports.node}" ]; }];
-					}
-					{
-						job_name = "smartctl";
-						# Querying SMART is comparatively slow and the data barely moves, so
-						# this job opts out of the global 15s. Safe to do here because the
-						# smartctl dashboard uses no rate() queries at all.
-						scrape_interval = "60s";
-						scrape_timeout = "30s";
-						static_configs = [{ targets = [ "${hostName}:${toString ports.smartctl}" ]; }];
-					}
-				];
-			};
-
-			# ---------------------------------------------------------------
-			# Grafana
-			# https://wiki.nixos.org/wiki/Grafana
-			# ---------------------------------------------------------------
-			services.grafana = {
-				enable = true;
-				settings.server = { http_addr = "0.0.0.0"; http_port = ports.grafana; };
-
-				provision = {
-					enable = true;
-
-					# Mutable dashboard provider pulling from /etc/grafana-dashboards, so
-					# dashboards can be declared with environment.etc below.
-					dashboards.settings.providers = [{
-						name = "my dashboards";
-						disableDeletion = true;
-						options = {
-							path = "/etc/grafana-dashboards";
-							foldersFromFilesStructure = true;
-						};
-					}];
-
-					# Grafana cannot change the UID of an existing datasource in place. Its
-					# provisioner finds the datasource by name, then calls UpdateDataSource,
-					# which re-fetches using `id AND uid` — with a new uid that row doesn't
-					# match, so it fails with "data source not found" and Grafana refuses to
-					# start. Deleting first turns the upsert into a plain insert; the
-					# provisioner runs deletes before inserts for exactly this case, and
-					# suppresses the deletion event because the datasource is about to be
-					# re-created. Safe to leave in permanently: both datasources are fully
-					# declarative, and dashboards bind to them by UID, not by database ID.
-					datasources.settings.deleteDatasources = [
-						{ name = "Prometheus"; orgId = 1; }
-						{ name = "Loki"; orgId = 1; }
-					];
-
-					datasources.settings.datasources = [
-						{
-							name = "Prometheus";
-							type = "prometheus";
-							uid = dsUid.prometheus;  # fixed, so dashboards can reference it
-							url = "http://127.0.0.1:${toString ports.prometheus}";
-							isDefault = true;
-							jsonData = {
-								# Tells Grafana how often the data is actually scraped. This is
-								# what $__rate_interval is derived from; leaving it unset makes
-								# Grafana guess 15s and mis-size every rate() window.
-								timeInterval = scrapeInterval;
-								httpMethod = "POST";
-							};
-						}
-						{
-							name = "Loki";
-							type = "loki";
-							uid = dsUid.loki;
-							url = "http://127.0.0.1:${toString ports.loki}";
-						}
-					];
-				};
-			};
-
-			# Grafana secret key
-			systemd.services.grafana.serviceConfig.LoadCredential = [
-				"secret_key:/run/secrets/grafana-secret_key"
-			];
-			services.grafana.settings.security.secret_key =
-				"$__file{/run/credentials/grafana.service/secret_key}";
-
-			# ---------------------------------------------------------------
-			# Dashboards
-			# ---------------------------------------------------------------
-			environment.etc = lib.mapAttrs' (name: drv:
-				lib.nameValuePair "grafana-dashboards/${name}.json" { source = drv; }
-			) dashboards;
-
-			# ---------------------------------------------------------------
-			# Reverse proxy
-			# ---------------------------------------------------------------
-			services.caddy = {
-				enable = true;
-				virtualHosts.":80".extraConfig = ''
-					reverse_proxy localhost:${toString ports.grafana}
-				'';
-			};
-
-			system.stateVersion = "26.05";
-		};
-	};
-}
diff --git a/modules/collectors.nix b/modules/collectors.nix
new file mode 100644
index 0000000..6f328ee
--- /dev/null
+++ b/modules/collectors.nix
@@ -0,0 +1,368 @@
+# Host-side telemetry sources.
+#
+# Everything here has to run on the host, not in a container: it reads host hardware and
+# kernel state -- /proc, /sys, SMART over raw block devices, ZFS ioctls -- that a
+# container has no view of. The half that can be containerised is modules/server.nix.
+{ config, lib, pkgs, ... }:
+
+let
+  inherit (lib) mkIf mkOption mkEnableOption mkForce types optionals concatMap;
+
+  cfg = config.services.telemetry.collectors;
+  ports = config.services.telemetry.ports;
+
+  collectorPkgs = pkgs.callPackages ./packages.nix {
+    # Match the userland to the running kernel module.
+    zfs = config.boot.zfs.package;
+  };
+
+  # services.zfs.autoSnapshot's per-label keep counts, handed to the collector so it can
+  # report how far each label is from its target rather than just how many snapshots
+  # exist. Silently empty when auto-snapshotting is off, which the collector handles.
+  snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ];
+
+  retentionArgs = optionals config.services.zfs.autoSnapshot.enable (
+    concatMap
+      (label: [
+        "--retention"
+        "${label}=${toString config.services.zfs.autoSnapshot.${label}}"
+      ])
+      snapshotLabels
+  );
+
+  # Hardening common to both collectors. Each writes exactly one file into the textfile
+  # directory and talks to nothing over the network.
+  hardening = {
+    LockPersonality = true;
+    MemoryDenyWriteExecute = true;
+    NoNewPrivileges = true;
+    PrivateNetwork = true;
+    PrivateTmp = true;
+    ProtectControlGroups = true;
+    ProtectHome = true;
+    ProtectHostname = true;
+    ProtectKernelLogs = true;
+    ProtectKernelModules = true;
+    ProtectSystem = "strict";
+    RestrictAddressFamilies = [ "AF_UNIX" ];
+    RestrictNamespaces = true;
+    RestrictRealtime = true;
+    RestrictSUIDSGID = true;
+    SystemCallArchitectures = "native";
+  };
+
+  mkCollectorService = { description, execStart, extraServiceConfig ? { } }: {
+    inherit description;
+    serviceConfig = hardening // {
+      Type = "oneshot";
+      ExecStart = execStart;
+      User = "root";
+      StateDirectory = "prometheus-node-exporter-textfile";
+      StateDirectoryMode = "0755";
+      # node_exporter runs as a different user and has to be able to read what we write.
+      UMask = "0022";
+      TimeoutStartSec = "180s";
+    } // extraServiceConfig;
+  };
+
+  mkCollectorTimer = { description, onBootSec, onUnitActiveSec, accuracySec, unit }: {
+    inherit description;
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnBootSec = onBootSec;
+      OnUnitActiveSec = onUnitActiveSec;
+      AccuracySec = accuracySec;
+      Unit = unit;
+    };
+  };
+in
+{
+  imports = [ ./common.nix ];
+
+  options.services.telemetry.collectors = {
+    enable = mkEnableOption "host-side telemetry collectors (node exporter, SMART, ZFS)";
+
+    listenAddress = mkOption {
+      type = types.str;
+      default = "0.0.0.0";
+      description = ''
+        Address the exporters bind to. The default is deliberately permissive because
+        the whole point of splitting the halves is that the server scrapes these from
+        somewhere else -- a container on a bridge, or another machine. Narrow it if the
+        server runs on this host.
+      '';
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Open the node and smartctl exporter ports. Needed whenever the server is not on
+        this host's loopback, which includes a container with `privateNetwork`.
+      '';
+    };
+
+    textfileDir = mkOption {
+      type = types.path;
+      default = "/var/lib/prometheus-node-exporter-textfile";
+      description = ''
+        Directory the ZFS and SMART collectors write `.prom` files into, and that
+        node_exporter's textfile collector reads. Changing this also changes the
+        StateDirectory of both collector units.
+      '';
+    };
+
+    node = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run the Prometheus node exporter.";
+      };
+
+      extraCollectors = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        example = [ "interrupts" ];
+        description = ''
+          Additional node_exporter collectors, on top of the systemd, processes and
+          textfile collectors this module enables.
+
+          Note this *adds* to node_exporter's defaults -- cpu, diskstats, filesystem,
+          meminfo, zfs, hwmon and pressure are already on, so listing those does
+          nothing. `interrupts` is the notable opt-in: it fills the Node Exporter Full
+          "IRQ Detail" panel but emits one series per (cpu, irq) pair, which is
+          thousands of series on a many-core box.
+        '';
+      };
+    };
+
+    smart = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Run the smartctl exporter and the self-test history collector.
+        '';
+      };
+
+      devices = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        example = [ "/dev/sda" "/dev/nvme0" ];
+        description = ''
+          Devices to poll. Empty means auto-discovery via `smartctl --scan`, which is
+          correct for direct-attached AHCI/HBA disks. Behind a RAID controller you
+          generally have to list devices explicitly with their `-d` type.
+        '';
+      };
+
+      maxInterval = mkOption {
+        type = types.str;
+        default = "60s";
+        description = ''
+          Floor on how often the exporter actually shells out to smartctl, independent
+          of how often Prometheus scrapes it. SMART data moves slowly and polling can
+          spin up idle disks.
+        '';
+      };
+
+      selfTestInterval = mkOption {
+        type = types.str;
+        default = "15min";
+        description = ''
+          How often to re-read the self-test logs. The data only changes when a test
+          finishes, so this can be far slower than the exporter's own polling.
+        '';
+      };
+    };
+
+    zfs = {
+      enable = mkOption {
+        type = types.bool;
+        default = config.boot.zfs.enabled;
+        defaultText = lib.literalExpression "config.boot.zfs.enabled";
+        description = ''
+          Collect pool capacity and health, vdev errors, scrub history, dataset
+          properties and snapshot retention from the zpool/zfs CLIs.
+        '';
+      };
+
+      interval = mkOption {
+        type = types.str;
+        default = "1min";
+        description = ''
+          How often to re-read pool and dataset state. Listing every snapshot on a busy
+          pool is the slow part; if this starts overlapping its own runs, raise it.
+        '';
+      };
+    };
+
+    logs = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Ship the systemd journal to Loki with Grafana Alloy.";
+      };
+
+      lokiUrl = mkOption {
+        type = types.str;
+        default = "http://localhost:${toString ports.loki}";
+        defaultText = lib.literalExpression ''"http://localhost:''${toString config.services.telemetry.ports.loki}"'';
+        example = "http://metrics.lan:3100";
+        description = ''
+          Base URL of the Loki instance to push to. Point this at wherever
+          modules/server.nix runs; the `/loki/api/v1/push` path is appended.
+        '';
+      };
+
+      maxAge = mkOption {
+        type = types.str;
+        default = "12h";
+        description = ''
+          How far back into the journal to read on startup. Anything older than this is
+          skipped rather than replayed into Loki.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # ------------------------------------------------------------------------------
+    # node exporter
+    # ------------------------------------------------------------------------------
+    services.prometheus.exporters.node = mkIf cfg.node.enable {
+      enable = true;
+      port = ports.node;
+      inherit (cfg) listenAddress;
+
+      enabledCollectors = [
+        "systemd" # node_systemd_* — the "Systemd" row, and the zfs-* timer panels
+        "processes" # node_processes_* — "Processes Detailed States", "PIDs", "Threads"
+      ] ++ cfg.node.extraCollectors;
+
+      extraFlags = [
+        "--collector.textfile.directory=${cfg.textfileDir}"
+        # node_systemd_unit_start_time_seconds, so the dashboards can show when the
+        # zfs-snapshot-* and zfs-scrub units last ran rather than only whether they
+        # failed.
+        "--collector.systemd.enable-start-time-metrics"
+      ];
+    };
+
+    # Upstream hardens the unit with ProtectHome=true, which makes /home look empty and
+    # blanks out the filesystem panels for any dataset mounted under it. read-only still
+    # allows statfs while keeping the sandbox.
+    systemd.services.prometheus-node-exporter = mkIf cfg.node.enable {
+      serviceConfig.ProtectHome = mkForce "read-only";
+    };
+
+    # ------------------------------------------------------------------------------
+    # SMART
+    # ------------------------------------------------------------------------------
+    services.prometheus.exporters.smartctl = mkIf cfg.smart.enable {
+      enable = true;
+      port = ports.smartctl;
+      inherit (cfg) listenAddress;
+      inherit (cfg.smart) devices maxInterval;
+    };
+
+    systemd.services.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorService {
+      description = "Collect SMART self-test history and drive identity";
+      execStart = "${lib.getExe collectorPkgs.smart-selftest} --output ${cfg.textfileDir}/smart-selftest.prom";
+      # Same shape as the smartctl exporter's own hardening: raw device access to talk
+      # to the disks, and nothing else.
+      extraServiceConfig = {
+        AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
+        CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ];
+        DevicePolicy = "closed";
+        DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ];
+      };
+    });
+
+    systemd.timers.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorTimer {
+      description = "Collect SMART self-test history periodically";
+      onBootSec = "3min";
+      onUnitActiveSec = cfg.smart.selfTestInterval;
+      accuracySec = "1min";
+      unit = "smart-selftest-collector.service";
+    });
+
+    # ------------------------------------------------------------------------------
+    # ZFS
+    # ------------------------------------------------------------------------------
+    systemd.services.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorService {
+      description = "Collect ZFS pool, dataset and snapshot metrics";
+      execStart = lib.escapeShellArgs ([
+        (lib.getExe collectorPkgs.zfs-metrics)
+        "--output"
+        "${cfg.textfileDir}/zfs.prom"
+      ] ++ retentionArgs);
+      # CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated on it;
+      # everything else is dropped.
+      extraServiceConfig = {
+        CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ];
+        SystemCallFilter = [ "@system-service" ];
+      };
+    });
+
+    systemd.timers.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorTimer {
+      description = "Collect ZFS metrics periodically";
+      onBootSec = "1min";
+      onUnitActiveSec = cfg.zfs.interval;
+      accuracySec = "5s";
+      unit = "zfs-textfile-collector.service";
+    });
+
+    # An earlier version of the ZFS collector wrote zpool.prom. Nothing removes a stale
+    # textfile once its writer is gone, and node_exporter will happily keep serving
+    # those frozen metrics forever, so delete it on activation.
+    systemd.tmpfiles.rules = mkIf cfg.zfs.enable [
+      "r ${cfg.textfileDir}/zpool.prom"
+    ];
+
+    # ------------------------------------------------------------------------------
+    # Journal shipping
+    # ------------------------------------------------------------------------------
+    services.alloy.enable = mkIf cfg.logs.enable true;
+
+    environment.etc."alloy/config.alloy" = mkIf cfg.logs.enable {
+      text = ''
+        loki.source.journal "journal" {
+          forward_to    = [loki.write.default.receiver]
+          relabel_rules = loki.relabel.journal.rules
+          max_age       = "${cfg.logs.maxAge}"
+        }
+
+        loki.relabel "journal" {
+          forward_to = []
+          rule {
+            source_labels = ["__journal__systemd_unit"]
+            target_label  = "unit"
+          }
+          rule {
+            source_labels = ["__journal__hostname"]
+            target_label  = "host"
+          }
+          rule {
+            source_labels = ["__journal_priority_keyword"]
+            target_label  = "level"
+          }
+        }
+
+        loki.write "default" {
+          endpoint {
+            url = "${cfg.logs.lokiUrl}/loki/api/v1/push"
+          }
+        }
+      '';
+    };
+
+    # ------------------------------------------------------------------------------
+    # Firewall
+    # ------------------------------------------------------------------------------
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall (
+      optionals cfg.node.enable [ ports.node ]
+      ++ optionals cfg.smart.enable [ ports.smartctl ]
+    );
+  };
+}
diff --git a/modules/common.nix b/modules/common.nix
new file mode 100644
index 0000000..480aa62
--- /dev/null
+++ b/modules/common.nix
@@ -0,0 +1,46 @@
+# Options both halves need to agree on.
+#
+# The collectors bind these ports; the server scrapes them and pushes to them. When the
+# two halves live in separate NixOS configurations (host + container) there is no shared
+# `config` to read, so each side sets its own copy and it is on you to keep them equal —
+# which is why they are gathered in one small file rather than scattered.
+#
+# Imported by both modules/collectors.nix and modules/server.nix. Importing the same
+# path twice is harmless: the module system deduplicates imports by path, so a
+# configuration that uses both halves does not end up declaring these twice.
+{ lib, ... }:
+
+let
+  inherit (lib) mkOption types;
+
+  portOption = default: description: mkOption {
+    inherit default description;
+    type = types.port;
+  };
+in
+{
+  options.services.telemetry.ports = {
+    node = portOption 9100 ''
+      Port the Prometheus node exporter listens on. Also where the server's `node`
+      scrape job points.
+    '';
+
+    smartctl = portOption 9633 ''
+      Port the smartctl exporter listens on. Also where the server's `smartctl` scrape
+      job points.
+    '';
+
+    loki = portOption 3100 ''
+      Port Loki listens on. Also where the collectors' log shipper pushes to.
+    '';
+
+    prometheus = portOption 9090 ''
+      Port Prometheus listens on.
+    '';
+
+    grafana = portOption 3000 ''
+      Port Grafana listens on. If the reverse proxy is enabled this stays internal and
+      the proxy is what you connect to.
+    '';
+  };
+}
diff --git a/modules/dashboards.nix b/modules/dashboards.nix
new file mode 100644
index 0000000..125e3e2
--- /dev/null
+++ b/modules/dashboards.nix
@@ -0,0 +1,75 @@
+# Grafana dashboards, built into a form file-provisioning can actually use.
+#
+# Returns an attrset of { name = derivation; }, where each derivation is a single
+# dashboard JSON file. modules/server.nix drops these into /etc/grafana-dashboards.
+{ lib, runCommand, fetchurl, jq, dsUid }:
+
+let
+  # Dashboards exported from grafana.com carry an `__inputs` block and reference their
+  # datasource as "''${DS_PROMETHEUS}". That placeholder is only substituted by Grafana's
+  # interactive *import* flow -- file provisioning never runs it, so the uid stays
+  # literal and every panel fails to resolve a datasource. Substitute at build time.
+  #
+  # A dashboard-level `datasource`-type template variable (lowercase, e.g.
+  # "''${ds_prometheus}") is a different mechanism and resolves fine on its own; this
+  # only touches names declared in __inputs.
+  fixupProgram = ''
+    def dsuid($pluginId):
+      if   $pluginId == "prometheus" then $prom
+      elif $pluginId == "loki"       then $loki
+      else "" end;
+
+    def subst($name; $uid):
+      walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end);
+
+    reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId)))
+    | del(.__inputs, .__requires)
+    # Provisioned dashboards must not carry an id, or Grafana rejects them on collision
+    # with whatever already occupies that id in its database.
+    | .id = null
+  '';
+
+  fixup = name: src: runCommand "grafana-dashboard-${name}.json"
+    { nativeBuildInputs = [ jq ]; }
+    ''
+      jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \
+         --arg loki ${lib.escapeShellArg dsUid.loki} \
+         ${lib.escapeShellArg fixupProgram} < ${src} > "$out"
+    '';
+
+  fromGrafanaCom = { name, id, rev, hash }:
+    fixup name (fetchurl {
+      url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download";
+      inherit hash;
+    });
+in
+{
+  # https://grafana.com/grafana/dashboards/1860
+  node-exporter-full = fromGrafanaCom {
+    name = "node-exporter-full";
+    id = 1860;
+    rev = 45;
+    hash = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY=";
+  };
+
+  # https://grafana.com/grafana/dashboards/22604
+  smartctl-exporter = fromGrafanaCom {
+    name = "smartctl-exporter";
+    id = 22604;
+    rev = 2;
+    hash = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88=";
+  };
+
+  # Dashboard 15362 ("ZFS Pool Metrics") used to fill this slot. It is an *InfluxDB*
+  # dashboard -- its queries are InfluxQL against a `zpool_stats` measurement -- so it
+  # can never render against Prometheus, however it is provisioned. This is a
+  # Prometheus-native replacement built on the metrics we actually export: node
+  # exporter's zfs collector plus the ZFS textfile collector.
+  # Generated by dashboards/gen_zfs.py.
+  zfs-pools = fixup "zfs-pools" ../dashboards/zfs-pools.json;
+
+  # CrystalDiskInfo-style per-drive SMART view. Generated by dashboards/gen_smart.py.
+  # The Good/Caution/Bad verdict and the per-drive row titles both come from the
+  # recording rules in rules/smartctl.yml, so this one needs those loaded to be useful.
+  drive-health = fixup "drive-health" ../dashboards/smart.json;
+}
diff --git a/modules/default.nix b/modules/default.nix
new file mode 100644
index 0000000..07aa854
--- /dev/null
+++ b/modules/default.nix
@@ -0,0 +1,13 @@
+# Both halves at once, for running collectors and the web services on one machine.
+#
+# Importing this is equivalent to importing collectors.nix and server.nix together. If
+# you are splitting them -- collectors on the host, server in a container -- import the
+# two modules separately instead; each is self-contained.
+{ ... }:
+
+{
+  imports = [
+    ./collectors.nix
+    ./server.nix
+  ];
+}
diff --git a/modules/packages.nix b/modules/packages.nix
new file mode 100644
index 0000000..5e76f10
--- /dev/null
+++ b/modules/packages.nix
@@ -0,0 +1,45 @@
+# The two textfile collectors, as ordinary packages.
+#
+# They are wrappers rather than inline `writeShellApplication` bodies in the module so
+# that the Python is runnable by hand -- `zfs-metrics --output /tmp/x.prom` behaves
+# exactly as the timer does, which is how every one of these metrics was validated.
+# Neither wrapper hardcodes an output path; the unit passes it.
+{ lib, writeShellApplication, python3, smartmontools, zfs }:
+
+{
+  # Pool capacity and health, vdev errors, scrub history, dataset properties and
+  # snapshot retention -- none of which node_exporter's built-in zfs collector reports,
+  # because it reads /proc/spl/kstat/zfs and that only carries ARC statistics and objset
+  # counters. OpenZFS 2.2+ also dropped the per-pool `io` kstat, so node_zfs_zpool_* is
+  # largely empty. This shells out to the zpool/zfs CLIs instead.
+  #
+  # runtimeInputs is the whole reason for the wrapper: the collector must run against
+  # the same zfs userland as the running kernel module.
+  zfs-metrics = writeShellApplication {
+    name = "zfs-metrics";
+    runtimeInputs = [ zfs ];
+    text = ''
+      exec ${python3.interpreter} ${../collectors/zfs-metrics.py} "$@"
+    '';
+    meta = {
+      description = "Export ZFS pool, dataset and snapshot metrics in Prometheus format";
+      mainProgram = "zfs-metrics";
+    };
+  };
+
+  # smartctl_exporter reads only the entry *count* out of each drive's self-test log and
+  # discards the log table, so it cannot say when a drive last ran a short or extended
+  # test or whether it passed. It also has no way to render a drive's identity as a
+  # display string, which the dashboards need for row titles. Both come from here.
+  smart-selftest = writeShellApplication {
+    name = "smart-selftest";
+    runtimeInputs = [ smartmontools ];
+    text = ''
+      exec ${python3.interpreter} ${../collectors/smart-selftest.py} "$@"
+    '';
+    meta = {
+      description = "Export SMART self-test history and drive identity in Prometheus format";
+      mainProgram = "smart-selftest";
+    };
+  };
+}
diff --git a/modules/server.nix b/modules/server.nix
new file mode 100644
index 0000000..1996f5e
--- /dev/null
+++ b/modules/server.nix
@@ -0,0 +1,420 @@
+# The web half: Prometheus, Loki, Grafana and an optional reverse proxy.
+#
+# Nothing here reads host hardware, so this is the part you can put in a container. It
+# reaches the collectors over the network -- see `collectorHosts`.
+{ config, lib, pkgs, ... }:
+
+let
+  inherit (lib) mkIf mkOption mkEnableOption mkDefault types optionals filterAttrs mapAttrs' nameValuePair;
+
+  cfg = config.services.telemetry.server;
+  ports = config.services.telemetry.ports;
+
+  # Fixed, because the generated dashboards reference them literally. dashboards/panels.py
+  # emits {"type": "prometheus", "uid": "prometheus"} on every panel, and the jq fixup in
+  # dashboards.nix rewrites grafana.com's placeholders to the same values. Changing these
+  # would silently break every panel, so they are not options.
+  dsUid = { prometheus = "prometheus"; loki = "loki"; };
+
+  # A `host` label carrying the machine name with the port stripped off.
+  #
+  # `instance` cannot do this job. The same machine is scraped twice on different ports
+  # -- the node exporter on :9100 serves the textfile metrics (zfs_*, smart_selftest_*,
+  # smart_drive_label) while smartctl_* comes from its own exporter on :9633 -- so one
+  # machine has two `instance` values, and the recording rules join straight across that
+  # boundary. `host` is equal on both, which is what makes the dashboards' host picker
+  # and those joins work at the same time.
+  #
+  # Note the regex is fully anchored by Prometheus, and only handles host:port. A
+  # bracketed IPv6 literal needs an explicit label via `extraScrapeConfigs` instead.
+  hostLabelRelabel = [{
+    source_labels = [ "__address__" ];
+    regex = "([^:]+)(?::[0-9]+)?";
+    target_label = "host";
+    replacement = "$1";
+  }];
+
+  builtinDashboards = pkgs.callPackage ./dashboards.nix { inherit dsUid; };
+
+  selected = filterAttrs (name: _: cfg.grafana.dashboards.${name} or false) builtinDashboards;
+
+  dashboards = selected // cfg.grafana.extraDashboards;
+
+  dashboardOption = description: mkOption {
+    inherit description;
+    type = types.bool;
+    default = true;
+  };
+in
+{
+  imports = [ ./common.nix ];
+
+  options.services.telemetry.server = {
+    enable = mkEnableOption "the Prometheus/Loki/Grafana server";
+
+    collectorHosts = mkOption {
+      type = types.listOf types.str;
+      default = [ "localhost" ];
+      example = [ "poweredge" "optiplex" ];
+      description = ''
+        Machines running modules/collectors.nix. Each is scraped on both the node and
+        smartctl ports from {option}`services.telemetry.ports`, so this is normally the
+        only thing on the server side that has to know about the hosts.
+
+        Each entry also becomes the value of the `host` label on everything scraped
+        from that machine, which is what the dashboards' host picker selects on. Use
+        the same spelling you would type in a browser; if you list an IP address, that
+        is what the picker will show.
+
+        Set {option}`services.telemetry.server.prometheus.nodeTargets` or
+        {option}`services.telemetry.server.prometheus.smartctlTargets` to take over
+        target generation entirely.
+      '';
+    };
+
+    listenAddress = mkOption {
+      type = types.str;
+      default = "0.0.0.0";
+      description = ''
+        Address Loki, Prometheus and Grafana bind to. Loki has to be reachable from
+        every host running the collectors, so this defaults to all interfaces.
+      '';
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Open the ports of whichever services are enabled: Loki (the collectors push to
+        it), Prometheus (useful for checking scrape health directly), and either the
+        proxy or Grafana.
+      '';
+    };
+
+    prometheus = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run Prometheus.";
+      };
+
+      scrapeInterval = mkOption {
+        type = types.str;
+        default = "15s";
+        description = ''
+          Global scrape interval, and the value handed to Grafana's Prometheus
+          datasource as `timeInterval`.
+
+          These two MUST agree. Grafana derives `$__rate_interval` from `timeInterval`,
+          and if it guesses wrong then `rate()` windows are mis-sized and every
+          rate-based panel intermittently renders "No data" -- which is exactly what
+          Prometheus' own 1m default does to the Node Exporter Full dashboard. Setting
+          it here sets both.
+        '';
+      };
+
+      nodeTargets = mkOption {
+        type = types.listOf types.str;
+        default = map (host: "${host}:${toString ports.node}") cfg.collectorHosts;
+        defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the node exporter port";
+        description = "Targets for the `node` scrape job.";
+      };
+
+      smartctlTargets = mkOption {
+        type = types.listOf types.str;
+        default = map (host: "${host}:${toString ports.smartctl}") cfg.collectorHosts;
+        defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the smartctl exporter port";
+        description = "Targets for the `smartctl` scrape job. Empty disables the job.";
+      };
+
+      smartctlScrapeInterval = mkOption {
+        type = types.str;
+        default = "60s";
+        description = ''
+          Scrape interval for the smartctl job, which opts out of the global one:
+          querying SMART is slow and the data barely moves. Safe because the drive
+          dashboards use no `rate()` queries at all.
+        '';
+      };
+
+      ruleFiles = mkOption {
+        type = types.listOf types.path;
+        default = [ ../rules/smartctl.yml ];
+        defaultText = lib.literalExpression "[ ../rules/smartctl.yml ]";
+        description = ''
+          Recording rule files. The default computes the per-drive Good/Caution/Bad
+          verdict and the row-title labels that the drive health dashboard displays, so
+          drop it only if you also drop that dashboard.
+
+          {option}`services.prometheus.checkConfig` runs promtool over these at build
+          time, so a broken rule fails the build rather than the service.
+        '';
+      };
+
+      extraScrapeConfigs = mkOption {
+        type = types.listOf types.attrs;
+        default = [ ];
+        description = "Additional scrape jobs, appended to the generated ones.";
+      };
+
+      retentionTime = mkOption {
+        type = types.str;
+        default = "90d";
+        description = "How long to keep samples.";
+      };
+    };
+
+    loki = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run Loki to receive journal logs from the collectors.";
+      };
+
+      dataDir = mkOption {
+        type = types.path;
+        default = "/var/lib/loki";
+        description = "Where Loki stores its chunks and index.";
+      };
+    };
+
+    grafana = {
+      enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Run Grafana, provisioned with datasources and dashboards.";
+      };
+
+      secretKeyFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        example = lib.literalExpression "config.sops.secrets.grafana-secret-key.path";
+        description = ''
+          File holding Grafana's `security.secret_key`. Loaded through systemd's
+          `LoadCredential`, so the file only has to be readable by root at start time
+          and never lands in the Nix store.
+
+          Deliberately a plain path rather than a sops-nix integration: wire it up to
+          whatever secret manager you already use.
+        '';
+      };
+
+      dashboards = {
+        node-exporter-full = dashboardOption "Provision grafana.com dashboard 1860 (Node Exporter Full).";
+        smartctl-exporter = dashboardOption "Provision grafana.com dashboard 22604 (Smartctl Exporter).";
+        zfs-pools = dashboardOption "Provision the ZFS pools dashboard. Needs the ZFS collector.";
+        drive-health = dashboardOption "Provision the CrystalDiskInfo-style drive health dashboard. Needs the SMART collector and the smartctl recording rules.";
+      };
+
+      extraDashboards = mkOption {
+        type = types.attrsOf types.path;
+        default = { };
+        example = lib.literalExpression ''{ my-dashboard = ./my-dashboard.json; }'';
+        description = ''
+          Extra dashboards to provision, as { name = path-to-json; }. These are passed
+          through unmodified, so they must already reference the datasource UIDs
+          "prometheus" and "loki".
+        '';
+      };
+    };
+
+    proxy = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Run Caddy in front of Grafana. Off by default because a reverse proxy is
+          usually something you already have; turn it on to get Grafana on port 80
+          without exposing its own port.
+        '';
+      };
+
+      virtualHost = mkOption {
+        type = types.str;
+        default = ":80";
+        example = "metrics.example.com";
+        description = ''
+          Caddy virtual host. A bare hostname makes Caddy provision a certificate
+          automatically; ":80" serves plain HTTP on every name.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # Grafana uses this key to encrypt datasource credentials in its database, and
+    # nixpkgs stopped shipping a default in 26.05. Caught here rather than left to
+    # Grafana's own assertion, which names an option this module manages for you.
+    assertions = [{
+      assertion = !cfg.grafana.enable || cfg.grafana.secretKeyFile != null;
+      message = ''
+        services.telemetry.server.grafana.secretKeyFile must be set when Grafana is
+        enabled. Generate a key (`openssl rand -base64 32`), store it with whatever
+        secret manager you use, and point this option at the resulting file.
+      '';
+    }];
+
+    # --------------------------------------------------------------------------------
+    # Loki
+    # --------------------------------------------------------------------------------
+    services.loki = mkIf cfg.loki.enable {
+      enable = true;
+      configuration = {
+        auth_enabled = false;
+
+        server = {
+          http_listen_port = ports.loki;
+          # Explicit: the collectors have to reach this from another host or container.
+          http_listen_address = cfg.listenAddress;
+        };
+
+        common = {
+          path_prefix = cfg.loki.dataDir;
+          storage.filesystem = {
+            chunks_directory = "${cfg.loki.dataDir}/chunks";
+            rules_directory = "${cfg.loki.dataDir}/rules";
+          };
+          ring = {
+            instance_addr = "127.0.0.1";
+            kvstore.store = "inmemory";
+          };
+          replication_factor = 1;
+        };
+
+        schema_config.configs = [{
+          from = "2024-01-01";
+          store = "tsdb";
+          object_store = "filesystem";
+          schema = "v13";
+          index = { prefix = "index_"; period = "24h"; };
+        }];
+      };
+    };
+
+    # --------------------------------------------------------------------------------
+    # Prometheus
+    # --------------------------------------------------------------------------------
+    services.prometheus = mkIf cfg.prometheus.enable {
+      enable = true;
+      port = ports.prometheus;
+      listenAddress = cfg.listenAddress;
+      retentionTime = cfg.prometheus.retentionTime;
+
+      globalConfig = {
+        scrape_interval = cfg.prometheus.scrapeInterval;
+        evaluation_interval = cfg.prometheus.scrapeInterval;
+        scrape_timeout = "10s";
+      };
+
+      inherit (cfg.prometheus) ruleFiles;
+
+      scrapeConfigs =
+        optionals (cfg.prometheus.nodeTargets != [ ]) [{
+          job_name = "node";
+          relabel_configs = hostLabelRelabel;
+          static_configs = [{ targets = cfg.prometheus.nodeTargets; }];
+        }]
+        ++ optionals (cfg.prometheus.smartctlTargets != [ ]) [{
+          job_name = "smartctl";
+          scrape_interval = cfg.prometheus.smartctlScrapeInterval;
+          scrape_timeout = "30s";
+          relabel_configs = hostLabelRelabel;
+          static_configs = [{ targets = cfg.prometheus.smartctlTargets; }];
+        }]
+        ++ cfg.prometheus.extraScrapeConfigs;
+    };
+
+    # --------------------------------------------------------------------------------
+    # Grafana
+    # --------------------------------------------------------------------------------
+    services.grafana = mkIf cfg.grafana.enable {
+      enable = true;
+
+      settings = {
+        server = {
+          http_addr = mkDefault cfg.listenAddress;
+          http_port = mkDefault ports.grafana;
+        };
+      }
+      # A file provider rather than a literal, so the key is never written into a
+      # world-readable store path. The credential is mounted by LoadCredential below.
+      // lib.optionalAttrs (cfg.grafana.secretKeyFile != null) {
+        security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}";
+      };
+
+      provision = {
+        enable = true;
+
+        dashboards.settings.providers = [{
+          name = "provisioned";
+          disableDeletion = true;
+          options = {
+            path = "/etc/grafana-dashboards";
+            foldersFromFilesStructure = true;
+          };
+        }];
+
+        # Grafana cannot change the UID of an existing datasource in place. Its
+        # provisioner finds the datasource by name, then calls UpdateDataSource, which
+        # re-fetches using `id AND uid` -- with a new uid that row does not match, so it
+        # fails with "data source not found" and Grafana refuses to start. Deleting
+        # first turns the upsert into a plain insert; the provisioner runs deletes
+        # before inserts for exactly this case and suppresses the deletion event because
+        # the datasource is about to be re-created. Safe permanently: both datasources
+        # are fully declarative and dashboards bind to them by UID, not database ID.
+        datasources.settings.deleteDatasources =
+          [{ name = "Prometheus"; orgId = 1; }]
+          ++ optionals cfg.loki.enable [{ name = "Loki"; orgId = 1; }];
+
+        datasources.settings.datasources =
+          optionals cfg.prometheus.enable [{
+            name = "Prometheus";
+            type = "prometheus";
+            uid = dsUid.prometheus;
+            url = "http://127.0.0.1:${toString ports.prometheus}";
+            isDefault = true;
+            jsonData = {
+              # What $__rate_interval is derived from. Leaving it unset makes Grafana
+              # assume 15s and mis-size every rate() window. See scrapeInterval above.
+              timeInterval = cfg.prometheus.scrapeInterval;
+              httpMethod = "POST";
+            };
+          }]
+          ++ optionals cfg.loki.enable [{
+            name = "Loki";
+            type = "loki";
+            uid = dsUid.loki;
+            url = "http://127.0.0.1:${toString ports.loki}";
+          }];
+      };
+    };
+
+    systemd.services.grafana = mkIf (cfg.grafana.enable && cfg.grafana.secretKeyFile != null) {
+      serviceConfig.LoadCredential = [ "secret_key:${cfg.grafana.secretKeyFile}" ];
+    };
+
+    environment.etc = mkIf cfg.grafana.enable (
+      mapAttrs' (name: src: nameValuePair "grafana-dashboards/${name}.json" { source = src; })
+        dashboards
+    );
+
+    # --------------------------------------------------------------------------------
+    # Reverse proxy
+    # --------------------------------------------------------------------------------
+    services.caddy = mkIf cfg.proxy.enable {
+      enable = true;
+      virtualHosts.${cfg.proxy.virtualHost}.extraConfig = ''
+        reverse_proxy localhost:${toString ports.grafana}
+      '';
+    };
+
+    # --------------------------------------------------------------------------------
+    # Firewall
+    # --------------------------------------------------------------------------------
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall (
+      optionals cfg.loki.enable [ ports.loki ]
+      ++ optionals cfg.prometheus.enable [ ports.prometheus ]
+      ++ (if cfg.proxy.enable then [ 80 443 ] else optionals cfg.grafana.enable [ ports.grafana ])
+    );
+  };
+}
diff --git a/rules/smartctl.yml b/rules/smartctl.yml
index dafed8d..3982e06 100644
--- a/rules/smartctl.yml
+++ b/rules/smartctl.yml
@@ -5,7 +5,7 @@
 #
 # The fleet is mixed: SATA drives expose the classic ATA attribute table, SAS drives
 # expose none of it and report health through SCSI error counters instead. Every rule
-# below is therefore made *total* -- it ends in `or (count by (device) (...) * 0)` so
+# below is therefore made *total* -- it ends in `or (count by (host, device) (...) * 0)` so
 # that each series exists with value 0 for every drive, including drives of the other
 # kind. Without that, the arithmetic in smartctl:health_code would silently drop drives
 # whose label sets did not match on both sides.
@@ -20,12 +20,12 @@ groups:
       # fails", so those are excluded rather than treated as an instant failure.
       - record: smartctl:attr_failing:count
         expr: |
-          count by (device) (
+          count by (host, device) (
             smartctl_device_attribute{attribute_value_type="value"}
-            <= on(device, attribute_id)
+            <= on(host, device, attribute_id)
             (smartctl_device_attribute{attribute_value_type="thresh"} > 0)
           )
-          or (count by (device) (smartctl_device_smart_status) * 0)
+          or (count by (host, device) (smartctl_device_smart_status) * 0)
 
       # Attributes whose raw value is a genuine warning sign across vendors:
       #   5   Reallocated_Sector_Ct     187 Reported_Uncorrect
@@ -35,13 +35,13 @@ groups:
       # values on Seagate drives by design and would flag healthy disks.
       - record: smartctl:attr_caution:count
         expr: |
-          count by (device) (
+          count by (host, device) (
             smartctl_device_attribute{
               attribute_value_type="raw",
               attribute_id=~"5|187|188|197|198|199"
             } > 0
           )
-          or (count by (device) (smartctl_device_smart_status) * 0)
+          or (count by (host, device) (smartctl_device_smart_status) * 0)
 
       # --- SAS / SCSI -----------------------------------------------------------
       # Grown defect list: sectors reallocated since the drive left the factory. The
@@ -49,15 +49,15 @@ groups:
       # acting on.
       - record: smartctl:scsi_defects:count
         expr: |
-          sum by (device) (smartctl_scsi_grown_defect_list)
-          or (count by (device) (smartctl_device_smart_status) * 0)
+          sum by (host, device) (smartctl_scsi_grown_defect_list)
+          or (count by (host, device) (smartctl_device_smart_status) * 0)
 
       # Errors the drive could not correct at all, which means data was lost.
       - record: smartctl:scsi_uncorrected:count
         expr: |
-          sum by (device) (smartctl_read_total_uncorrected_errors)
-          + sum by (device) (smartctl_write_total_uncorrected_errors)
-          or (count by (device) (smartctl_device_smart_status) * 0)
+          sum by (host, device) (smartctl_read_total_uncorrected_errors)
+          + sum by (host, device) (smartctl_write_total_uncorrected_errors)
+          or (count by (host, device) (smartctl_device_smart_status) * 0)
 
       # --- Verdict --------------------------------------------------------------
       # 0 good, 1 caution, 2 bad. The inner clamp_max on each group keeps two
@@ -66,7 +66,7 @@ groups:
         expr: |
           clamp_max(
             2 * clamp_max(
-                  (max by (device) (smartctl_device_smart_status) == bool 0)
+                  (max by (host, device) (smartctl_device_smart_status) == bool 0)
                 + (smartctl:attr_failing:count > bool 0)
                 + (smartctl:scsi_uncorrected:count > bool 0)
               , 1)
@@ -82,9 +82,9 @@ groups:
       # table, and only SCSI drives report a grown defect list.
       - record: smartctl:bus
         expr: |
-          label_replace(count by (device) (smartctl_device_attribute), "bus", "ata", "", "")
+          label_replace(count by (host, device) (smartctl_device_attribute), "bus", "ata", "", "")
           or
-          label_replace(count by (device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "")
+          label_replace(count by (host, device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "")
 
       # The identity string from collectors/smart-selftest.py, made total so a drive
       # still gets a row if that collector has not written its file yet. `unless` (not
@@ -92,11 +92,11 @@ groups:
       # `or` would emit two series per drive rather than deduplicating.
       - record: smartctl:ident
         expr: |
-          max by (device, label) (smart_drive_label)
+          max by (host, device, label) (smart_drive_label)
           or
           label_replace(
-            count by (device) (smartctl_device_smart_status)
-            unless on(device) smart_drive_label,
+            count by (host, device) (smartctl_device_smart_status)
+            unless on(host, device) smart_drive_label,
             "label", "identity pending", "", "")
 
       # One preformatted line per drive: "sde · GOOD · 6.0 TB · SAS 7.2K · model · SN".
@@ -111,6 +111,6 @@ groups:
              or label_replace(smartctl:health_code == 1, "verdict", "CAUTION", "", "")
              or label_replace(smartctl:health_code == 2, "verdict", "BAD", "", "")
             )
-            * on(device) group_left(bus) smartctl:bus
-            * on(device) group_left(label) smartctl:ident,
+            * on(host, device) group_left(bus) smartctl:bus
+            * on(host, device) group_left(label) smartctl:ident,
             "row", " · ", "device", "verdict", "label")
-- 
cgit v1.3.1