split into a flake with separable collectors/server modules

Replaces the single lgtm.nix with a flake exposing two NixOS modules under
services.telemetry, so the web services can move into a container declared in
the consuming config rather than one hardcoded here.

  modules/collectors.nix  host-side: node + smartctl exporters, the ZFS and
                          SMART textfile collector timers, Alloy journal
                          shipping. Must run on the host -- reads /proc, /sys,
                          raw block devices and ZFS ioctls.
  modules/server.nix      Prometheus, Loki, Grafana, optional Caddy.
  modules/common.nix      ports, shared by both halves.
  modules/dashboards.nix  jq fixup and grafana.com fetches.
  modules/packages.nix    the two collectors as ordinary packages.

Modules are exported as paths, not functions, so they evaluate with the
consumer's nixpkgs; the flake's own input only affects packages/checks/devShells.

Multi-host support. collectorHosts is a list, both scrape jobs relabel
__address__ into a `host` label, and all 133 queries in the two generated
dashboards are scoped to host=~"$host" behind a single-select Host picker.
`instance` could not serve as that key: one machine is scraped twice on
different ports (textfile metrics on :9100, smartctl_* on :9633) and the
recording rules join across that boundary.

Grafana's secret key becomes a plain secretKeyFile path wired through
LoadCredential, with an assertion -- nixpkgs 26.05 dropped the default. No
sops-nix input.

nix flake check runs promtool over the rules, diffs the committed dashboard
JSON against what the generators produce, and evaluates both modules.

Also fixes smart-selftest emitting a placeholder identity line when smartctl
cannot open a device. smartctl exits 0 with well-formed but empty JSON in that
case, so "SSD - unknown model - no serial" was reaching a dashboard row title;
it now emits nothing and lets the smartctl:ident fallback apply.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ

1 files changed, 85 insertions, 64 deletions

diff --git a/dashboards/smart.json b/dashboards/smart.json
index ed48cd1..f6c9867 100644
--- a/dashboards/smart.json
+++ b/dashboards/smart.json
@@ -20,6 +20,27 @@
   "templating": {
     "list": [
       {
+        "name": "host",
+        "label": "Host",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "query": {
+          "qryType": 1,
+          "query": "label_values(smartctl_device_smart_status, host)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "includeAll": false,
+        "multi": false,
+        "options": [],
+        "sort": 1,
+        "hide": 0,
+        "regex": ""
+      },
+      {
         "name": "ata_device",
         "label": "SATA drive",
         "type": "query",
@@ -29,7 +50,7 @@
         },
         "query": {
           "qryType": 1,
-          "query": "label_values(smartctl:row{bus=\"ata\"}, row)",
+          "query": "label_values(smartctl:row{bus=\"ata\", host=~\"$host\"}, row)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
@@ -59,7 +80,7 @@
         },
         "query": {
           "qryType": 1,
-          "query": "label_values(smartctl:row{bus=\"sas\"}, row)",
+          "query": "label_values(smartctl:row{bus=\"sas\", host=~\"$host\"}, row)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
@@ -166,7 +187,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smartctl:health_code == 0) or vector(0)",
+          "expr": "count(smartctl:health_code{host=~\"$host\"} == 0) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -233,7 +254,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smartctl:health_code == 1) or vector(0)",
+          "expr": "count(smartctl:health_code{host=~\"$host\"} == 1) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -300,7 +321,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smartctl:health_code == 2) or vector(0)",
+          "expr": "count(smartctl:health_code{host=~\"$host\"} == 2) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -363,7 +384,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smartctl_device_smart_status)",
+          "expr": "count(smartctl_device_smart_status{host=~\"$host\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -435,7 +456,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max(smartctl_device_temperature{temperature_type=\"current\"})",
+          "expr": "max(smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -499,7 +520,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)))",
+          "expr": "max(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)))",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -664,7 +685,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl:health_code * on(device) group_left(model_name, serial_number, protocol) smartctl_device",
+          "expr": "smartctl:health_code{host=~\"$host\"} * on(device) group_left(model_name, serial_number, protocol) smartctl_device{host=~\"$host\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -676,7 +697,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_temperature{temperature_type=\"current\"})",
+          "expr": "max by (device) (smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"})",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -688,7 +709,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600))",
+          "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600))",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -700,7 +721,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_capacity_bytes)",
+          "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\"})",
           "refId": "D",
           "instant": true,
           "range": false,
@@ -841,7 +862,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_temperature{temperature_type=\"current\"}",
+          "expr": "smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -948,7 +969,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl:health_code{device=~\"$ata_device\"}",
+          "expr": "smartctl:health_code{host=~\"$host\", device=~\"$ata_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1020,7 +1041,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_temperature{device=~\"$ata_device\", temperature_type=\"current\"}",
+          "expr": "smartctl_device_temperature{host=~\"$host\", device=~\"$ata_device\", temperature_type=\"current\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1084,7 +1105,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$ata_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$ata_device\"} * 3600))",
+          "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\", host=~\"$host\", device=~\"$ata_device\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\", host=~\"$host\", device=~\"$ata_device\"} * 3600))",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1148,7 +1169,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_capacity_bytes{device=~\"$ata_device\"}",
+          "expr": "smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$ata_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1211,7 +1232,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_power_cycle_count{device=~\"$ata_device\"}",
+          "expr": "smartctl_device_power_cycle_count{host=~\"$host\", device=~\"$ata_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1278,7 +1299,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"5\", attribute_value_type=\"raw\"}",
+          "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_id=\"5\", attribute_value_type=\"raw\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1345,7 +1366,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"197\", attribute_value_type=\"raw\"}",
+          "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_id=\"197\", attribute_value_type=\"raw\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1442,7 +1463,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device{device=~\"$ata_device\"}",
+          "expr": "smartctl_device{host=~\"$host\", device=~\"$ata_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1454,7 +1475,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$ata_device\"})",
+          "expr": "max by (device) (smartctl_device_rotation_rate{host=~\"$host\", device=~\"$ata_device\"})",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -1466,7 +1487,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$ata_device\"})",
+          "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$ata_device\"})",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -1623,7 +1644,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"}",
+          "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"value\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1635,7 +1656,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"worst\"})",
+          "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"worst\"})",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -1647,7 +1668,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})",
+          "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"thresh\"})",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -1659,7 +1680,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"raw\"})",
+          "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"raw\"})",
           "refId": "D",
           "instant": true,
           "range": false,
@@ -1671,7 +1692,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"} - on(device, attribute_id) smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})",
+          "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"value\"} - on(device, attribute_id) smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"thresh\"})",
           "refId": "E",
           "instant": true,
           "range": false,
@@ -1831,7 +1852,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl:health_code{device=~\"$sas_device\"}",
+          "expr": "smartctl:health_code{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1903,7 +1924,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_temperature{device=~\"$sas_device\", temperature_type=\"current\"}",
+          "expr": "smartctl_device_temperature{host=~\"$host\", device=~\"$sas_device\", temperature_type=\"current\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -1967,7 +1988,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$sas_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$sas_device\"} * 3600))",
+          "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\", host=~\"$host\", device=~\"$sas_device\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\", host=~\"$host\", device=~\"$sas_device\"} * 3600))",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2031,7 +2052,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_capacity_bytes{device=~\"$sas_device\"}",
+          "expr": "smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2095,7 +2116,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device_interface_speed{device=~\"$sas_device\", speed_type=\"current\"} * 1000",
+          "expr": "smartctl_device_interface_speed{host=~\"$host\", device=~\"$sas_device\", speed_type=\"current\"} * 1000",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2162,7 +2183,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}",
+          "expr": "smartctl_scsi_grown_defect_list{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2229,7 +2250,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"} + smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}",
+          "expr": "smartctl_read_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"} + smartctl_write_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2326,7 +2347,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_device{device=~\"$sas_device\"}",
+          "expr": "smartctl_device{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2338,7 +2359,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$sas_device\"})",
+          "expr": "max by (device) (smartctl_device_rotation_rate{host=~\"$host\", device=~\"$sas_device\"})",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -2350,7 +2371,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$sas_device\"})",
+          "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$sas_device\"})",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -2485,7 +2506,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_read_errors_corrected_by_eccfast{device=~\"$sas_device\"}",
+          "expr": "smartctl_read_errors_corrected_by_eccfast{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -2497,7 +2518,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_read_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}",
+          "expr": "smartctl_read_errors_corrected_by_eccdelayed{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "B",
           "instant": false,
           "range": true,
@@ -2509,7 +2530,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_read_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}",
+          "expr": "smartctl_read_errors_corrected_by_rereads_rewrites{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "C",
           "instant": false,
           "range": true,
@@ -2521,7 +2542,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_write_errors_corrected_by_eccfast{device=~\"$sas_device\"}",
+          "expr": "smartctl_write_errors_corrected_by_eccfast{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "D",
           "instant": false,
           "range": true,
@@ -2533,7 +2554,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_write_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}",
+          "expr": "smartctl_write_errors_corrected_by_eccdelayed{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "E",
           "instant": false,
           "range": true,
@@ -2545,7 +2566,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_write_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}",
+          "expr": "smartctl_write_errors_corrected_by_rereads_rewrites{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "F",
           "instant": false,
           "range": true,
@@ -2631,7 +2652,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}",
+          "expr": "smartctl_scsi_grown_defect_list{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "A",
           "instant": false,
           "range": true,
@@ -2643,7 +2664,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"}",
+          "expr": "smartctl_read_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "B",
           "instant": false,
           "range": true,
@@ -2655,7 +2676,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}",
+          "expr": "smartctl_write_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}",
           "refId": "C",
           "instant": false,
           "range": true,
@@ -2735,7 +2756,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smartctl_device_smart_status) - count(count by (device) (smart_selftest_last_lifetime_hours)) or count(smartctl_device_smart_status)",
+          "expr": "count(smartctl_device_smart_status{host=~\"$host\"}) - count(count by (device) (smart_selftest_last_lifetime_hours{host=~\"$host\"})) or count(smartctl_device_smart_status{host=~\"$host\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2802,7 +2823,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "count(smart_selftest_last_passed == 0) or vector(0)",
+          "expr": "count(smart_selftest_last_passed{host=~\"$host\"} == 0) or vector(0)",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2875,7 +2896,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{test_type=\"extended\"})",
+          "expr": "max(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{host=~\"$host\", test_type=\"extended\"})",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2979,7 +3000,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "label_join(smart_selftest_last_passed, \"drive_test\", \"/\", \"device\", \"test_type\")",
+          "expr": "label_join(smart_selftest_last_passed{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\")",
           "refId": "A",
           "instant": true,
           "range": false,
@@ -2991,7 +3012,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (drive_test) (label_join(smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))",
+          "expr": "max by (drive_test) (label_join(smart_selftest_last_lifetime_hours{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))",
           "refId": "B",
           "instant": true,
           "range": false,
@@ -3003,7 +3024,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (drive_test) (label_join(smart_selftest_count, \"drive_test\", \"/\", \"device\", \"test_type\"))",
+          "expr": "max by (drive_test) (label_join(smart_selftest_count{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))",
           "refId": "C",
           "instant": true,
           "range": false,
@@ -3015,7 +3036,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "max by (drive_test) (label_join(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))",
+          "expr": "max by (drive_test) (label_join(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))",
           "refId": "D",
           "instant": true,
           "range": false,
@@ -3166,7 +3187,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_attribute{attribute_id=\"5\", attribute_value_type=\"raw\"}",
+              "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=\"5\", attribute_value_type=\"raw\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3252,7 +3273,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_attribute{attribute_id=~\"197|198\", attribute_value_type=\"raw\"}",
+              "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=~\"197|198\", attribute_value_type=\"raw\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3338,7 +3359,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_attribute{attribute_id=\"199\", attribute_value_type=\"raw\"}",
+              "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=\"199\", attribute_value_type=\"raw\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3425,7 +3446,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_bytes_written",
+              "expr": "smartctl_device_bytes_written{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3437,7 +3458,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_bytes_read",
+              "expr": "smartctl_device_bytes_read{host=~\"$host\"}",
               "refId": "B",
               "instant": false,
               "range": true,
@@ -3545,7 +3566,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl:health_code",
+              "expr": "smartctl:health_code{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3645,7 +3666,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_device_smartctl_exit_status",
+              "expr": "smartctl_device_smartctl_exit_status{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3732,7 +3753,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smart_selftest_collection_duration_seconds",
+              "expr": "smart_selftest_collection_duration_seconds{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3818,7 +3839,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smartctl_devices",
+              "expr": "smartctl_devices{host=~\"$host\"}",
               "refId": "A",
               "instant": false,
               "range": true,
@@ -3830,7 +3851,7 @@
                 "uid": "prometheus"
               },
               "editorMode": "code",
-              "expr": "smart_selftest_devices",
+              "expr": "smart_selftest_devices{host=~\"$host\"}",
               "refId": "B",
               "instant": false,
               "range": true,