diff options
| -rw-r--r-- | README.md | 280 | ||||
| -rw-r--r-- | collectors/smart-selftest.py | 18 | ||||
| -rw-r--r-- | dashboards/gen_smart.py | 92 | ||||
| -rw-r--r-- | dashboards/gen_zfs.py | 76 | ||||
| -rw-r--r-- | dashboards/smart.json | 149 | ||||
| -rw-r--r-- | dashboards/zfs-pools.json | 173 | ||||
| -rw-r--r-- | flake.lock | 27 | ||||
| -rw-r--r-- | flake.nix | 109 | ||||
| -rw-r--r-- | lgtm.nix | 534 | ||||
| -rw-r--r-- | modules/collectors.nix | 368 | ||||
| -rw-r--r-- | modules/common.nix | 46 | ||||
| -rw-r--r-- | modules/dashboards.nix | 75 | ||||
| -rw-r--r-- | modules/default.nix | 13 | ||||
| -rw-r--r-- | modules/packages.nix | 45 | ||||
| -rw-r--r-- | modules/server.nix | 420 | ||||
| -rw-r--r-- | rules/smartctl.yml | 38 |
16 files changed, 1692 insertions, 771 deletions
diff --git a/README.md b/README.md new file mode 100644 index 0000000..7bcea7f --- /dev/null +++ b/README.md @@ -0,0 +1,280 @@ +# telemetry + +Host telemetry collection and a Prometheus/Loki/Grafana server, as two NixOS modules you +can deploy together or apart. + +--- + +## ⚠️ This repository is vibe coded. Run it at your own risk. + +Every line here — the modules, the collectors, the dashboard generators, the recording +rules, and this README — was written by an LLM in conversation. No human has reviewed it +line by line. There is no warranty of any kind. **Read it before you deploy it**, and pay +particular attention to the systemd hardening options, the firewall rules, and anything +touching raw block devices. + +What *has* been checked: + +- every dashboard query executed against a real Prometheus fed fixture data +- `promtool` over the recording rules and the generated scrape config +- `nix flake check`: both modules evaluate, and the committed dashboard JSON matches what + the generators produce +- the collectors run against mock `smartctl`/`zpool`/`zfs`, and partially against real + hardware + +What has **not**: + +- no panel has been rendered in a real Grafana +- no output has come from a real ZFS pool +- no real SAS self-test log has been parsed + +Treat the numbers on the dashboards as unverified until you have sanity-checked them +against `zpool status` and `smartctl -x` yourself. + +--- + +## What it is + +The split is not cosmetic. The collectors read host hardware and kernel state — `/proc`, +`/sys`, SMART over raw block devices, ZFS ioctls — which a container has no view of. The +server reads none of that, so it can live wherever you like. + +| module | contains | runs where | +|---|---|---| +| `nixosModules.collectors` | node + smartctl exporters, ZFS and SMART textfile collectors, Alloy | the host, necessarily | +| `nixosModules.server` | Prometheus, Loki, Grafana, optional Caddy | anywhere that can reach the collectors | +| `nixosModules.default` | both | one machine | + +Grafana comes provisioned with four dashboards: Node Exporter Full (1860), Smartctl +Exporter (22604), and two generated here — ZFS Pools and Drive Health, a +CrystalDiskInfo-style per-drive view. + +```nix +{ + inputs.telemetry.url = "path:/home/timmy/docs/src/flakes/lgtm"; +} +``` + +The examples below take `inputs` as a module argument, which means passing it through: + +```nix +nixosConfigurations.poweredge = nixpkgs.lib.nixosSystem { + specialArgs = { inherit inputs; }; + # ... +}; +``` + +--- + +## Example: one machine + +```nix +{ inputs, ... }: +{ + imports = [ inputs.telemetry.nixosModules.default ]; + + services.telemetry = { + collectors.enable = true; + server = { + enable = true; + grafana.secretKeyFile = "/run/secrets/grafana-secret-key"; + proxy.enable = true; # Grafana on :80 + }; + }; +} +``` + +Grafana at `http://localhost`. `collectorHosts` defaults to `[ "localhost" ]`. + +## Example: web services in a container + +The host runs the collectors; the container runs everything else. This is the intended +shape. + +```nix +{ config, inputs, ... }: +{ + imports = [ inputs.telemetry.nixosModules.collectors ]; + + services.telemetry.collectors = { + enable = true; + logs.lokiUrl = "http://metrics:3100"; # the container, by name + }; + + sops.secrets.grafana-secret-key = { + sopsFile = ./secrets/grafana.yaml; + key = "secret_key"; + }; + + containers.metrics = { + autoStart = true; + privateNetwork = true; + hostBridge = "br-lan0"; + localMacAddress = "02:00:00:00:00:06"; + + # The module reads whatever path you point secretKeyFile at. + bindMounts."/run/grafana-secret-key" = { + hostPath = config.sops.secrets.grafana-secret-key.path; + isReadOnly = true; + }; + + config = { ... }: { + imports = [ inputs.telemetry.nixosModules.server ]; + + networking.enableIPv6 = false; + networking.interfaces.eth0.useDHCP = true; + + services.telemetry.server = { + enable = true; + collectorHosts = [ "poweredge" ]; # the host above + grafana.secretKeyFile = "/run/grafana-secret-key"; + proxy.enable = true; + }; + + system.stateVersion = "26.05"; # the module does not set this for you + }; + }; +} +``` + +Two names have to resolve for this to work: the container must be able to look up +`poweredge`, and the host must be able to look up `metrics`. With `hostBridge` plus DHCP +on the LAN that is your router's job; otherwise use IP addresses in both places. + +## Example: several machines + +List them, and import the collectors module on each: + +```nix +# in the container +services.telemetry.server.collectorHosts = [ "poweredge" "optiplex" ]; +``` + +```nix +# on optiplex +{ inputs, ... }: +{ + imports = [ inputs.telemetry.nixosModules.collectors ]; + + services.telemetry.collectors = { + enable = true; + logs.lokiUrl = "http://metrics:3100"; + }; +} +``` + +You get **one set of dashboards with a Host dropdown**, not a set per machine. Pick a +host and the whole page swaps to it. + +| dashboard | picker(s) | +|---|---| +| Drive Health | **Host** → SATA drive / SAS drive | +| ZFS Pools | **Host** → Pool | +| Node Exporter Full | Job → Nodename → Instance | +| Smartctl Exporter | Node | + +Dependent dropdowns repopulate on switch, so you never see poweredge's `tank` while +viewing optiplex. Hosts appear and disappear on their own — the pickers are +`label_values(..., host)`, so a machine shows up as soon as Prometheus scrapes it. + +The Host picker is **single-select with no "All"** on purpose: drive and pool names are +not unique across a fleet, and two hosts at once would make `max by (device)` silently +merge two different physical drives into one plausible-looking row. + +## Example: turning things off + +```nix +services.telemetry = { + collectors = { + enable = true; + zfs.enable = false; # no ZFS on this box + logs.enable = false; # no journal shipping + node.extraCollectors = [ "interrupts" ]; # fills 1860's IRQ panel; high cardinality + }; + + server = { + enable = true; + loki.enable = false; + grafana.dashboards.zfs-pools = false; + grafana.extraDashboards.mine = ./my-dashboard.json; + prometheus.extraScrapeConfigs = [{ + job_name = "router"; + static_configs = [{ targets = [ "router:9100" ]; }]; + }]; + }; +}; +``` + +Ports live in `services.telemetry.ports` (`node`, `smartctl`, `loki`, `prometheus`, +`grafana`) and are shared by both halves. + +Jobs added through `extraScrapeConfigs` are passed to Prometheus untouched — they get no +`host` label, so they will not show up in the dashboards' Host picker. Add your own +`relabel_configs` if you want them to. + +--- + +## Layout + +``` +flake.nix modules, collector packages, checks, dev shell +modules/ + common.nix ports, shared by both halves + collectors.nix host-side exporters and timers + server.nix Prometheus, Loki, Grafana, proxy + dashboards.nix builds dashboard JSON into a provisionable form + packages.nix the two collectors as ordinary packages +collectors/*.py the collectors themselves; runnable by hand +dashboards/gen_*.py dashboard generators +dashboards/*.json generated, committed, checked +rules/smartctl.yml Prometheus recording rules +``` + +## Working on it + +Dashboard JSON is **generated** — edit `dashboards/gen_*.py`, never the JSON: + +```sh +python3 dashboards/gen_zfs.py > dashboards/zfs-pools.json +python3 dashboards/gen_smart.py > dashboards/smart.json +``` + +`nix flake check` fails if you forget. + +The collectors are plain scripts taking `--output PATH`, so you can see exactly what a +machine reports: + +```sh +nix run .#zfs-metrics -- --output /tmp/zfs.prom +sudo nix run .#smart-selftest -- --output /tmp/smart.prom +``` + +`smart-selftest` needs privileges to open the devices. Without them smartctl still exits +0 with well-formed but empty JSON; the collector logs `no identity data` rather than +inventing placeholder values. + +## Things that will bite you + +- **`prometheus.scrapeInterval` deliberately sets two things.** Grafana derives + `$__rate_interval` from the datasource's `timeInterval`; if that disagrees with the real + scrape interval, `rate()` windows are mis-sized and rate-based panels intermittently + render "No data". Don't configure the Grafana datasource by hand. +- **The datasource UIDs `prometheus` and `loki` are fixed, not options.** The generated + dashboards reference them literally. +- **Grafana's secret key is mandatory** as of nixpkgs 26.05. `openssl rand -base64 32`, + store it somewhere, point `grafana.secretKeyFile` at it. +- **Changing `collectorHosts` changes the label set.** Panels look empty for a minute + after first deploy while the recording rules re-evaluate; data from before the change + stays under the old labels. +- **smartd does not run self-tests just because it is enabled.** NixOS's default `-a` + flags contain no `-s` schedule, so self-test history stays empty: + ```nix + services.smartd.defaults.autodetected = "-a -o on -s (S/../.././02|L/../../6/03)"; + ``` +- **SAS drives have no power-on-hours attribute.** smartctl sources `power_on_time` for + SCSI from the Background Scan Results log page, which is not the drive's lifetime meter + and often reads far too low. Drive Health takes the larger of that and the lifetime + stamp in the drive's own self-test log — so it can exceed what `smartctl -x` prints. +- **IPv6 literals need help.** The `host` label is built by stripping the port off + `__address__` with `([^:]+)(?::[0-9]+)?`, which a bracketed IPv6 address does not match. + Use `prometheus.extraScrapeConfigs` with an explicit label for those. diff --git a/collectors/smart-selftest.py b/collectors/smart-selftest.py index fa9d68f..6cb026c 100644 --- a/collectors/smart-selftest.py +++ b/collectors/smart-selftest.py @@ -231,13 +231,25 @@ def collect_identity(reg: Registry, name: str, data: dict) -> None: This exists to be read by a human in a Grafana row title, so it is deliberately a single rendered string rather than separate labels -- Grafana can interpolate a variable's display text into a title, but cannot assemble one from several labels. + + Emits nothing unless smartctl actually returned identifying data. When it cannot + open the device -- no privileges, a controller that needs an explicit -d type -- it + still exits 0 with a well-formed JSON document that simply has no identity fields, + and filling those in with placeholders would put "unknown model · no serial" in a + dashboard row title. Emitting nothing instead lets the `smartctl:ident` recording + rule fall back to "identity pending", which is the truth. """ model = (data.get("model_name") or data.get("scsi_model_name") - or (data.get("scsi_product") or "") or "unknown model") - serial = data.get("serial_number") or "no serial" + or data.get("scsi_product") or "") + serial = data.get("serial_number") or "" capacity = human_capacity((data.get("user_capacity") or {}).get("bytes")) - parts = [p for p in (capacity, technology(data), model, serial) if p] + if not (model or serial or capacity): + warn(f"{name}: no identity data; is smartctl able to open the device?") + return + + parts = [p for p in (capacity, technology(data), model or "unknown model", + serial or "no serial") if p] reg.add("smart_drive_label", 1, {"device": name, "label": " · ".join(parts)}, "Drive identity rendered for display; the value is always 1.") diff --git a/dashboards/gen_smart.py b/dashboards/gen_smart.py index 2cc0568..de9ab34 100644 --- a/dashboards/gen_smart.py +++ b/dashboards/gen_smart.py @@ -54,8 +54,14 @@ PASS_MAP = [{ }, }] -ATA = 'device=~"$ata_device"' -SAS = 'device=~"$sas_device"' +# Every query on this dashboard is scoped to exactly one machine. The host variable is +# deliberately single-select with no "All": `sda` is not a unique name across a fleet, +# and the `max by (device)` reductions and joinByField transformations below would +# silently merge two different physical drives into one row if more than one host were +# ever selected at a time. +HOST = 'host=~"$host"' +ATA = f'{HOST}, device=~"$ata_device"' +SAS = f'{HOST}, device=~"$sas_device"' # Splits "sde · GOOD · 6.0 TB · SAS 7.2K · HGST HUS726060AL5210 · NAHPJS45" into a # selectable value (the leading device name) and display text (the whole line). The @@ -80,7 +86,7 @@ def power_on(sel=""): source is larger is therefore never an over-estimate, and recovers a sane number on drives whose power-on counter under-counts or was reset. """ - braces = f"{{{sel}}}" if sel else "" + braces = "{" + ", ".join(x for x in (HOST, sel) if x) + "}" return (f'max by (device) (smartctl_device_power_on_seconds{braces} ' f'or (smart_selftest_last_lifetime_hours{braces} * 3600))') @@ -124,14 +130,14 @@ panels = [] panels.append(row("Fleet Overview", 0)) panels.append(stat( - "Healthy", [('count(smartctl:health_code == 0) or vector(0)', "drives")], + "Healthy", [(f'count(smartctl:health_code{{{HOST}}} == 0) or vector(0)', "drives")], gp(0, 1, 4, 4), colormode="background", thresholds={"mode": "absolute", "steps": [{"color": "green", "value": None}]}, desc="Drives with no failing attribute, no SCSI uncorrected errors and a passing " "SMART self-assessment.", )) panels.append(stat( - "Caution", [('count(smartctl:health_code == 1) or vector(0)', "drives")], + "Caution", [(f'count(smartctl:health_code{{{HOST}}} == 1) or vector(0)', "drives")], gp(4, 1, 4, 4), colormode="background", thresholds={"mode": "absolute", "steps": [ {"color": "green", "value": None}, {"color": "orange", "value": 1}]}, @@ -139,18 +145,19 @@ panels.append(stat( "defect list on SAS. Not failing yet — worth watching the trend.", )) panels.append(stat( - "Failing", [('count(smartctl:health_code == 2) or vector(0)', "drives")], + "Failing", [(f'count(smartctl:health_code{{{HOST}}} == 2) or vector(0)', "drives")], gp(8, 1, 4, 4), colormode="background", thresholds=OK_BAD, desc="SMART self-assessment failed, an attribute is at or below its threshold, or " "the drive reported errors it could not correct. Replace these.", )) panels.append(stat( - "Drives", [('count(smartctl_device_smart_status)', "total")], + "Drives", [(f'count(smartctl_device_smart_status{{{HOST}}})', "total")], gp(12, 1, 4, 4), )) panels.append(stat( "Hottest Drive", - [('max(smartctl_device_temperature{temperature_type="current"})', "max")], + [(f'max(smartctl_device_temperature{{{HOST}, temperature_type="current"}})', + "max")], gp(16, 1, 4, 4), unit="celsius", colormode="background", thresholds={"mode": "absolute", "steps": [ {"color": "green", "value": None}, @@ -167,12 +174,12 @@ panels.append(table( "All Drives", [ # group_left carries the descriptive labels from the info metric onto the # verdict, so this one query supplies device, model, serial and protocol. - ('smartctl:health_code * on(device) group_left(model_name, serial_number, ' - 'protocol) smartctl_device', None), - ('max by (device) (smartctl_device_temperature{temperature_type="current"})', - None), + (f'smartctl:health_code{{{HOST}}} * on(device) group_left(model_name, ' + f'serial_number, protocol) smartctl_device{{{HOST}}}', None), + (f'max by (device) (smartctl_device_temperature{{{HOST}, ' + f'temperature_type="current"}})', None), (power_on(), None), - ('max by (device) (smartctl_device_capacity_bytes)', None), + (f'max by (device) (smartctl_device_capacity_bytes{{{HOST}}})', None), ], gp(0, 5, 16, 10), keep="device|model_name|serial_number|protocol|Value.*", renames={ @@ -204,7 +211,8 @@ panels.append(table( panels.append(timeseries( "Temperatures", - [('smartctl_device_temperature{temperature_type="current"}', "{{device}}")], + [(f'smartctl_device_temperature{{{HOST}, temperature_type="current"}}', + "{{device}}")], gp(16, 5, 8, 10), unit="celsius", desc="Sustained operation above ~45C measurably shortens drive life.", )) @@ -418,22 +426,23 @@ panels.append(row("Self-Tests", 53)) panels.append(stat( "Drives Never Tested", - [('count(smartctl_device_smart_status) - ' - 'count(count by (device) (smart_selftest_last_lifetime_hours)) or ' - 'count(smartctl_device_smart_status)', "drives")], + [(f'count(smartctl_device_smart_status{{{HOST}}}) - ' + f'count(count by (device) (smart_selftest_last_lifetime_hours{{{HOST}}})) or ' + f'count(smartctl_device_smart_status{{{HOST}}})', "drives")], gp(0, 54, 4, 5), thresholds=OK_BAD, colormode="background", desc="Drives with no self-test log entries at all. smartd only runs tests if its " "config includes an -s schedule; merely enabling the service does not.", )) panels.append(stat( "Failed Self-Tests", - [('count(smart_selftest_last_passed == 0) or vector(0)', "tests")], + [(f'count(smart_selftest_last_passed{{{HOST}}} == 0) or vector(0)', "tests")], gp(4, 54, 4, 5), thresholds=OK_BAD, colormode="background", )) panels.append(stat( "Oldest Long Test", [(f'max({power_on()} / 3600 - on(device) group_right() ' - f'smart_selftest_last_lifetime_hours{{test_type="extended"}})', "hours ago")], + f'smart_selftest_last_lifetime_hours{{{HOST}, test_type="extended"}})', + "hours ago")], gp(8, 54, 4, 5), unit="h", thresholds={"mode": "absolute", "steps": [ {"color": "green", "value": None}, @@ -461,14 +470,14 @@ def _keyed(expr): panels.append(table( "Self-Test History", [ # Only this query keeps device and test_type as their own columns. - ('label_join(smart_selftest_last_passed, "drive_test", "/", "device", ' - '"test_type")', None), - (_keyed('smart_selftest_last_lifetime_hours'), None), - (_keyed('smart_selftest_count'), None), + (f'label_join(smart_selftest_last_passed{{{HOST}}}, "drive_test", "/", ' + f'"device", "test_type")', None), + (_keyed(f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None), + (_keyed(f'smart_selftest_count{{{HOST}}}'), None), # Powered-on hours elapsed since that test ran. group_right() keeps the # right-hand side's labels, so device and test_type survive the subtraction. (_keyed(power_on() + ' / 3600 - on(device) group_right() ' - 'smart_selftest_last_lifetime_hours'), None), + f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None), ], gp(12, 54, 12, 10), keep="drive_test|device|test_type|Value.*", renames={ @@ -501,22 +510,22 @@ panels.append(table( panels.append(row("Attribute Trends", 64, collapsed=True, children=[ timeseries( "Reallocated Sectors (all SATA drives)", - [('smartctl_device_attribute{attribute_id="5", attribute_value_type="raw"}', - "{{device}}")], + [(f'smartctl_device_attribute{{{HOST}, attribute_id="5", ' + f'attribute_value_type="raw"}}', "{{device}}")], gp(0, 65, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, ), timeseries( "Pending & Uncorrectable Sectors", - [('smartctl_device_attribute{attribute_id=~"197|198", ' - 'attribute_value_type="raw"}', "{{device}} {{attribute_name}}")], + [(f'smartctl_device_attribute{{{HOST}, attribute_id=~"197|198", ' + f'attribute_value_type="raw"}}', "{{device}} {{attribute_name}}")], gp(8, 65, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, ), timeseries( "CRC Errors (cabling)", - [('smartctl_device_attribute{attribute_id="199", attribute_value_type="raw"}', - "{{device}}")], + [(f'smartctl_device_attribute{{{HOST}, attribute_id="199", ' + f'attribute_value_type="raw"}}', "{{device}}")], gp(16, 65, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, desc="Attribute 199 counts errors on the SATA link rather than the platters. " @@ -524,14 +533,14 @@ panels.append(row("Attribute Trends", 64, collapsed=True, children=[ ), timeseries( "Data Written / Read (SAS)", [ - ('smartctl_device_bytes_written', "{{device}} written"), - ('smartctl_device_bytes_read', "{{device}} read"), + (f'smartctl_device_bytes_written{{{HOST}}}', "{{device}} written"), + (f'smartctl_device_bytes_read{{{HOST}}}', "{{device}} read"), ], gp(0, 72, 12, 7), unit="bytes", minval=0, ), timeseries( "Health Code History", - [('smartctl:health_code', "{{device}}")], + [(f'smartctl:health_code{{{HOST}}}', "{{device}}")], gp(12, 72, 12, 7), mappings=HEALTH_MAP, minval=0, maxval=2, custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"}, ), @@ -540,7 +549,7 @@ panels.append(row("Attribute Trends", 64, collapsed=True, children=[ panels.append(row("Collector Health", 65, collapsed=True, children=[ timeseries( "smartctl Exit Status", - [('smartctl_device_smartctl_exit_status', "{{device}}")], + [(f'smartctl_device_smartctl_exit_status{{{HOST}}}', "{{device}}")], gp(0, 66, 8, 7), minval=0, desc="Non-zero means smartctl itself had trouble reading the device. Bit 0 is " "a command-line problem, bit 1 a device-open failure, bit 3 a failing " @@ -548,13 +557,13 @@ panels.append(row("Collector Health", 65, collapsed=True, children=[ ), timeseries( "Self-Test Collector Duration", - [('smart_selftest_collection_duration_seconds', "collection")], + [(f'smart_selftest_collection_duration_seconds{{{HOST}}}', "collection")], gp(8, 66, 8, 7), unit="s", minval=0, ), timeseries( "Devices Seen", [ - ('smartctl_devices', "exporter"), - ('smart_selftest_devices', "self-test collector"), + (f'smartctl_devices{{{HOST}}}', "exporter"), + (f'smart_selftest_devices{{{HOST}}}', "self-test collector"), ], gp(16, 66, 8, 7), minval=0, desc="These should agree. A divergence means one of the two cannot see some " @@ -575,9 +584,14 @@ print(json.dumps(dashboard( # `device=~"$ata_device"` selector below is unaffected, while the *text* the UI # shows -- and that ${ata_device:text} puts in the row title -- is the whole # line. That is what makes a collapsed row identifiable without opening it. - query_var("ata_device", 'label_values(smartctl:row{bus="ata"}, row)', + # One machine at a time. Not multi-select and no "All" -- see the comment + # on HOST above for why selecting two hosts at once would silently merge + # same-named drives rather than showing both. + query_var("host", "label_values(smartctl_device_smart_status, host)", + label="Host", multi=False, include_all=False), + query_var("ata_device", 'label_values(smartctl:row{bus="ata", host=~"$host"}, row)', label="SATA drive", regex=ROW_REGEX), - query_var("sas_device", 'label_values(smartctl:row{bus="sas"}, row)', + query_var("sas_device", 'label_values(smartctl:row{bus="sas", host=~"$host"}, row)', label="SAS drive", regex=ROW_REGEX), ], tags=["smart", "storage", "hardware"], diff --git a/dashboards/gen_zfs.py b/dashboards/gen_zfs.py index 4edb38b..0057506 100644 --- a/dashboards/gen_zfs.py +++ b/dashboards/gen_zfs.py @@ -20,7 +20,8 @@ If you add a panel, add the metric to the collector first and keep this list hon import json -from panels import (DS, OK_BAD, bargauge, gauge, gp, row, stat, table, timeseries) +from panels import (DS, OK_BAD, bargauge, gauge, gp, query_var, row, stat, table, + timeseries) STATE_MAP = [{ @@ -46,7 +47,12 @@ BOOL_MAP = [{ }, }] -POOL = 'pool=~"$pool"' +# Every query on this dashboard is scoped to exactly one machine. The host variable is +# single-select with no "All" on purpose: pool names are not unique across a fleet, and +# the `max by (pool)` reductions and joinByField transformations below would silently +# merge two different pools that happen to share a name. +HOST = 'host=~"$host"' +POOL = f'{HOST}, pool=~"$pool"' panels = [] # ====================================================================================== @@ -124,7 +130,7 @@ panels.append(stat( panels.append(stat( "Snapshots Overdue", [(f'count((time() - zfs_snapshot_newest_timestamp_seconds{{{POOL}}}) > on(label) ' - f'group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)', "overdue")], + f'group_left() (2 * zfs_snapshot_interval_seconds{{{HOST}}})) or vector(0)', "overdue")], gp(15, 5, 4, 4), thresholds=OK_BAD, colormode="background", desc="Dataset/label pairs whose newest snapshot is more than two intervals old — " "i.e. the auto-snapshot timer has missed at least one run.", @@ -294,7 +300,7 @@ panels.append(bargauge( panels.append(bargauge( "Snapshots vs Retention Target", [(f'min by (label) (zfs_snapshot_count{{{POOL}}} / on(label) group_left() ' - f'zfs_snapshot_retention_target)', "{{label}}")], + f'zfs_snapshot_retention_target{{{HOST}}})', "{{label}}")], gp(8, 43, 8, 8), unit="percentunit", maxv=1.2, thresholds={"mode": "absolute", "steps": [ {"color": "red", "value": None}, @@ -329,15 +335,16 @@ panels.append(timeseries( panels.append(stat( "Snapshot / Scrub Job Failures", - [('sum(node_systemd_unit_state{name=~"zfs-(snapshot|scrub).*",state="failed"}) ' - 'or vector(0)', "failed units")], + [(f'sum(node_systemd_unit_state{{{HOST}, name=~"zfs-(snapshot|scrub).*", ' + f'state="failed"}}) or vector(0)', "failed units")], gp(0, 59, 6, 6), thresholds=OK_BAD, colormode="background", desc="systemd units for the auto-snapshot and auto-scrub timers currently in a " "failed state.", )) panels.append(bargauge( "Time Since Timer Last Fired", - [('time() - node_systemd_timer_last_trigger_seconds{name=~"zfs-.*"}', "{{name}}")], + [(f'time() - node_systemd_timer_last_trigger_seconds{{{HOST}, ' + f'name=~"zfs-.*"}}', "{{name}}")], gp(6, 59, 18, 6), unit="s", desc="When each zfs-* systemd timer last triggered. This is the job side of the " "picture; the snapshot ages above are the result side.", @@ -445,9 +452,9 @@ panels.append(row("Performance", 91)) panels.append(timeseries( "Dataset Throughput", [ - (f'rate(node_zfs_zpool_dataset_nread{{zpool=~"$pool"}}[$__rate_interval])', + (f'rate(node_zfs_zpool_dataset_nread{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} read"), - (f'-rate(node_zfs_zpool_dataset_nwritten{{zpool=~"$pool"}}[$__rate_interval])', + (f'-rate(node_zfs_zpool_dataset_nwritten{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} write"), ], gp(0, 92, 12, 8), unit="Bps", @@ -456,9 +463,9 @@ panels.append(timeseries( )) panels.append(timeseries( "Dataset IOPS", [ - (f'rate(node_zfs_zpool_dataset_reads{{zpool=~"$pool"}}[$__rate_interval])', + (f'rate(node_zfs_zpool_dataset_reads{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} read"), - (f'-rate(node_zfs_zpool_dataset_writes{{zpool=~"$pool"}}[$__rate_interval])', + (f'-rate(node_zfs_zpool_dataset_writes{{{HOST}, zpool=~"$pool"}}[$__rate_interval])', "{{zpool}}/{{dataset}} write"), ], gp(12, 92, 12, 8), unit="iops", @@ -467,24 +474,24 @@ panels.append(timeseries( panels.append(timeseries( "ARC Size", [ - ("node_zfs_arc_size", "size"), - ("node_zfs_arc_c_max", "target max (c_max)"), - ("node_zfs_arc_c_min", "target min (c_min)"), + (f"node_zfs_arc_size{{{HOST}}}", "size"), + (f"node_zfs_arc_c_max{{{HOST}}}", "target max (c_max)"), + (f"node_zfs_arc_c_min{{{HOST}}}", "target min (c_min)"), ], gp(0, 100, 8, 8), unit="bytes", minval=0, )) panels.append(timeseries( "ARC Hit Ratio", - [("rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min(" - "rate(node_zfs_arc_hits[$__rate_interval]) + " - "rate(node_zfs_arc_misses[$__rate_interval]), 1)", "hit ratio")], + [(f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval]) / clamp_min(" + f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval]) + " + f"rate(node_zfs_arc_misses{{{HOST}}}[$__rate_interval]), 1)", "hit ratio")], gp(8, 100, 8, 8), unit="percentunit", minval=0, desc="clamp_min avoids a divide-by-zero while the pool is idle.", )) panels.append(timeseries( "ARC Hits / Misses", [ - ("rate(node_zfs_arc_hits[$__rate_interval])", "hits"), - ("rate(node_zfs_arc_misses[$__rate_interval])", "misses"), + (f"rate(node_zfs_arc_hits{{{HOST}}}[$__rate_interval])", "hits"), + (f"rate(node_zfs_arc_misses{{{HOST}}}[$__rate_interval])", "misses"), ], gp(16, 100, 8, 8), unit="ops", minval=0, )) @@ -553,7 +560,7 @@ panels.append(row("Pool Internals", 108, collapsed=True, children=[ panels.append(row("Collector Health", 109, collapsed=True, children=[ timeseries( "ZFS Collector Duration", - [("zfs_metrics_collection_duration_seconds", "collection")], + [(f"zfs_metrics_collection_duration_seconds{{{HOST}}}", "collection")], gp(0, 110, 8, 7), unit="s", minval=0, desc="How long collectors/zfs-metrics.py takes. Listing every snapshot is the " "slow part; if this approaches the 1 minute timer interval, raise " @@ -561,7 +568,7 @@ panels.append(row("Collector Health", 109, collapsed=True, children=[ ), timeseries( "Textfile Age", - [("time() - node_textfile_mtime_seconds", "{{file}}")], + [(f"time() - node_textfile_mtime_seconds{{{HOST}}}", "{{file}}")], gp(8, 110, 8, 7), unit="s", minval=0, desc="Age of each .prom file. If this climbs without bound the collector has " "stopped writing and every ZFS metric above is stale.", @@ -569,7 +576,7 @@ panels.append(row("Collector Health", 109, collapsed=True, children=[ timeseries( "Scrape Health", [ ('up{job="node"}', "node exporter up"), - ("node_textfile_scrape_error", "textfile parse error"), + (f"node_textfile_scrape_error{{{HOST}}}", "textfile parse error"), ], gp(16, 110, 8, 7), minval=0, desc="node_textfile_scrape_error goes to 1 if a .prom file is malformed, in " @@ -591,21 +598,16 @@ dashboard = { "graphTooltip": 1, "refresh": "1m", "time": {"from": "now-24h", "to": "now"}, - "templating": {"list": [{ - "name": "pool", - "label": "Pool", - "type": "query", - "datasource": DS, - "query": {"qryType": 1, "query": "label_values(zfs_pool_health_code, pool)", - "refId": "PrometheusVariableQueryEditor-VariableQuery"}, - "refresh": 1, - "includeAll": True, - "allValue": ".*", - "multi": True, - "current": {"text": ["All"], "value": ["$__all"]}, - "options": [], - "sort": 1, - }]}, + "templating": {"list": [ + # One machine at a time: not multi-select and no "All". See the comment on HOST + # above for why two hosts at once would silently merge same-named pools. + query_var("host", "label_values(zfs_pool_health_code, host)", + label="Host", multi=False, include_all=False), + # Pools are listed for the selected host only, so switching hosts re-populates + # the picker rather than offering pools that do not exist on this machine. + query_var("pool", 'label_values(zfs_pool_health_code{host=~"$host"}, pool)', + label="Pool"), + ]}, "annotations": {"list": [{ "builtIn": 1, "datasource": {"type": "datasource", "uid": "grafana"}, "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/dashboards/smart.json b/dashboards/smart.json index ed48cd1..f6c9867 100644 --- a/dashboards/smart.json +++ b/dashboards/smart.json @@ -20,6 +20,27 @@ "templating": { "list": [ { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(smartctl_device_smart_status, host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [], + "sort": 1, + "hide": 0, + "regex": "" + }, + { "name": "ata_device", "label": "SATA drive", "type": "query", @@ -29,7 +50,7 @@ }, "query": { "qryType": 1, - "query": "label_values(smartctl:row{bus=\"ata\"}, row)", + "query": "label_values(smartctl:row{bus=\"ata\", host=~\"$host\"}, row)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -59,7 +80,7 @@ }, "query": { "qryType": 1, - "query": "label_values(smartctl:row{bus=\"sas\"}, row)", + "query": "label_values(smartctl:row{bus=\"sas\", host=~\"$host\"}, row)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -166,7 +187,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smartctl:health_code == 0) or vector(0)", + "expr": "count(smartctl:health_code{host=~\"$host\"} == 0) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -233,7 +254,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smartctl:health_code == 1) or vector(0)", + "expr": "count(smartctl:health_code{host=~\"$host\"} == 1) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -300,7 +321,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smartctl:health_code == 2) or vector(0)", + "expr": "count(smartctl:health_code{host=~\"$host\"} == 2) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -363,7 +384,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smartctl_device_smart_status)", + "expr": "count(smartctl_device_smart_status{host=~\"$host\"})", "refId": "A", "instant": true, "range": false, @@ -435,7 +456,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max(smartctl_device_temperature{temperature_type=\"current\"})", + "expr": "max(smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"})", "refId": "A", "instant": true, "range": false, @@ -499,7 +520,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)))", + "expr": "max(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)))", "refId": "A", "instant": true, "range": false, @@ -664,7 +685,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl:health_code * on(device) group_left(model_name, serial_number, protocol) smartctl_device", + "expr": "smartctl:health_code{host=~\"$host\"} * on(device) group_left(model_name, serial_number, protocol) smartctl_device{host=~\"$host\"}", "refId": "A", "instant": true, "range": false, @@ -676,7 +697,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_temperature{temperature_type=\"current\"})", + "expr": "max by (device) (smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"})", "refId": "B", "instant": true, "range": false, @@ -688,7 +709,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600))", + "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600))", "refId": "C", "instant": true, "range": false, @@ -700,7 +721,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_capacity_bytes)", + "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\"})", "refId": "D", "instant": true, "range": false, @@ -841,7 +862,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_temperature{temperature_type=\"current\"}", + "expr": "smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"}", "refId": "A", "instant": false, "range": true, @@ -948,7 +969,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl:health_code{device=~\"$ata_device\"}", + "expr": "smartctl:health_code{host=~\"$host\", device=~\"$ata_device\"}", "refId": "A", "instant": true, "range": false, @@ -1020,7 +1041,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_temperature{device=~\"$ata_device\", temperature_type=\"current\"}", + "expr": "smartctl_device_temperature{host=~\"$host\", device=~\"$ata_device\", temperature_type=\"current\"}", "refId": "A", "instant": true, "range": false, @@ -1084,7 +1105,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$ata_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$ata_device\"} * 3600))", + "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\", host=~\"$host\", device=~\"$ata_device\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\", host=~\"$host\", device=~\"$ata_device\"} * 3600))", "refId": "A", "instant": true, "range": false, @@ -1148,7 +1169,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_capacity_bytes{device=~\"$ata_device\"}", + "expr": "smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$ata_device\"}", "refId": "A", "instant": true, "range": false, @@ -1211,7 +1232,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_power_cycle_count{device=~\"$ata_device\"}", + "expr": "smartctl_device_power_cycle_count{host=~\"$host\", device=~\"$ata_device\"}", "refId": "A", "instant": true, "range": false, @@ -1278,7 +1299,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"5\", attribute_value_type=\"raw\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_id=\"5\", attribute_value_type=\"raw\"}", "refId": "A", "instant": true, "range": false, @@ -1345,7 +1366,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"197\", attribute_value_type=\"raw\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_id=\"197\", attribute_value_type=\"raw\"}", "refId": "A", "instant": true, "range": false, @@ -1442,7 +1463,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device{device=~\"$ata_device\"}", + "expr": "smartctl_device{host=~\"$host\", device=~\"$ata_device\"}", "refId": "A", "instant": true, "range": false, @@ -1454,7 +1475,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$ata_device\"})", + "expr": "max by (device) (smartctl_device_rotation_rate{host=~\"$host\", device=~\"$ata_device\"})", "refId": "B", "instant": true, "range": false, @@ -1466,7 +1487,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$ata_device\"})", + "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$ata_device\"})", "refId": "C", "instant": true, "range": false, @@ -1623,7 +1644,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"value\"}", "refId": "A", "instant": true, "range": false, @@ -1635,7 +1656,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"worst\"})", + "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"worst\"})", "refId": "B", "instant": true, "range": false, @@ -1647,7 +1668,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})", + "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"thresh\"})", "refId": "C", "instant": true, "range": false, @@ -1659,7 +1680,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"raw\"})", + "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"raw\"})", "refId": "D", "instant": true, "range": false, @@ -1671,7 +1692,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"} - on(device, attribute_id) smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})", + "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"value\"} - on(device, attribute_id) smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"thresh\"})", "refId": "E", "instant": true, "range": false, @@ -1831,7 +1852,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl:health_code{device=~\"$sas_device\"}", + "expr": "smartctl:health_code{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": true, "range": false, @@ -1903,7 +1924,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_temperature{device=~\"$sas_device\", temperature_type=\"current\"}", + "expr": "smartctl_device_temperature{host=~\"$host\", device=~\"$sas_device\", temperature_type=\"current\"}", "refId": "A", "instant": true, "range": false, @@ -1967,7 +1988,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$sas_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$sas_device\"} * 3600))", + "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\", host=~\"$host\", device=~\"$sas_device\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\", host=~\"$host\", device=~\"$sas_device\"} * 3600))", "refId": "A", "instant": true, "range": false, @@ -2031,7 +2052,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_capacity_bytes{device=~\"$sas_device\"}", + "expr": "smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": true, "range": false, @@ -2095,7 +2116,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_interface_speed{device=~\"$sas_device\", speed_type=\"current\"} * 1000", + "expr": "smartctl_device_interface_speed{host=~\"$host\", device=~\"$sas_device\", speed_type=\"current\"} * 1000", "refId": "A", "instant": true, "range": false, @@ -2162,7 +2183,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}", + "expr": "smartctl_scsi_grown_defect_list{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": true, "range": false, @@ -2229,7 +2250,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"} + smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}", + "expr": "smartctl_read_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"} + smartctl_write_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": true, "range": false, @@ -2326,7 +2347,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device{device=~\"$sas_device\"}", + "expr": "smartctl_device{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": true, "range": false, @@ -2338,7 +2359,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$sas_device\"})", + "expr": "max by (device) (smartctl_device_rotation_rate{host=~\"$host\", device=~\"$sas_device\"})", "refId": "B", "instant": true, "range": false, @@ -2350,7 +2371,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$sas_device\"})", + "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$sas_device\"})", "refId": "C", "instant": true, "range": false, @@ -2485,7 +2506,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_read_errors_corrected_by_eccfast{device=~\"$sas_device\"}", + "expr": "smartctl_read_errors_corrected_by_eccfast{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": false, "range": true, @@ -2497,7 +2518,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_read_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}", + "expr": "smartctl_read_errors_corrected_by_eccdelayed{host=~\"$host\", device=~\"$sas_device\"}", "refId": "B", "instant": false, "range": true, @@ -2509,7 +2530,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_read_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}", + "expr": "smartctl_read_errors_corrected_by_rereads_rewrites{host=~\"$host\", device=~\"$sas_device\"}", "refId": "C", "instant": false, "range": true, @@ -2521,7 +2542,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_write_errors_corrected_by_eccfast{device=~\"$sas_device\"}", + "expr": "smartctl_write_errors_corrected_by_eccfast{host=~\"$host\", device=~\"$sas_device\"}", "refId": "D", "instant": false, "range": true, @@ -2533,7 +2554,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_write_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}", + "expr": "smartctl_write_errors_corrected_by_eccdelayed{host=~\"$host\", device=~\"$sas_device\"}", "refId": "E", "instant": false, "range": true, @@ -2545,7 +2566,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_write_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}", + "expr": "smartctl_write_errors_corrected_by_rereads_rewrites{host=~\"$host\", device=~\"$sas_device\"}", "refId": "F", "instant": false, "range": true, @@ -2631,7 +2652,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}", + "expr": "smartctl_scsi_grown_defect_list{host=~\"$host\", device=~\"$sas_device\"}", "refId": "A", "instant": false, "range": true, @@ -2643,7 +2664,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"}", + "expr": "smartctl_read_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}", "refId": "B", "instant": false, "range": true, @@ -2655,7 +2676,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}", + "expr": "smartctl_write_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}", "refId": "C", "instant": false, "range": true, @@ -2735,7 +2756,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smartctl_device_smart_status) - count(count by (device) (smart_selftest_last_lifetime_hours)) or count(smartctl_device_smart_status)", + "expr": "count(smartctl_device_smart_status{host=~\"$host\"}) - count(count by (device) (smart_selftest_last_lifetime_hours{host=~\"$host\"})) or count(smartctl_device_smart_status{host=~\"$host\"})", "refId": "A", "instant": true, "range": false, @@ -2802,7 +2823,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(smart_selftest_last_passed == 0) or vector(0)", + "expr": "count(smart_selftest_last_passed{host=~\"$host\"} == 0) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -2875,7 +2896,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{test_type=\"extended\"})", + "expr": "max(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{host=~\"$host\", test_type=\"extended\"})", "refId": "A", "instant": true, "range": false, @@ -2979,7 +3000,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "label_join(smart_selftest_last_passed, \"drive_test\", \"/\", \"device\", \"test_type\")", + "expr": "label_join(smart_selftest_last_passed{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\")", "refId": "A", "instant": true, "range": false, @@ -2991,7 +3012,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (drive_test) (label_join(smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))", + "expr": "max by (drive_test) (label_join(smart_selftest_last_lifetime_hours{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))", "refId": "B", "instant": true, "range": false, @@ -3003,7 +3024,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (drive_test) (label_join(smart_selftest_count, \"drive_test\", \"/\", \"device\", \"test_type\"))", + "expr": "max by (drive_test) (label_join(smart_selftest_count{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))", "refId": "C", "instant": true, "range": false, @@ -3015,7 +3036,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (drive_test) (label_join(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))", + "expr": "max by (drive_test) (label_join(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))", "refId": "D", "instant": true, "range": false, @@ -3166,7 +3187,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{attribute_id=\"5\", attribute_value_type=\"raw\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=\"5\", attribute_value_type=\"raw\"}", "refId": "A", "instant": false, "range": true, @@ -3252,7 +3273,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{attribute_id=~\"197|198\", attribute_value_type=\"raw\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=~\"197|198\", attribute_value_type=\"raw\"}", "refId": "A", "instant": false, "range": true, @@ -3338,7 +3359,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_attribute{attribute_id=\"199\", attribute_value_type=\"raw\"}", + "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=\"199\", attribute_value_type=\"raw\"}", "refId": "A", "instant": false, "range": true, @@ -3425,7 +3446,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_bytes_written", + "expr": "smartctl_device_bytes_written{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3437,7 +3458,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_bytes_read", + "expr": "smartctl_device_bytes_read{host=~\"$host\"}", "refId": "B", "instant": false, "range": true, @@ -3545,7 +3566,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl:health_code", + "expr": "smartctl:health_code{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3645,7 +3666,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_device_smartctl_exit_status", + "expr": "smartctl_device_smartctl_exit_status{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3732,7 +3753,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smart_selftest_collection_duration_seconds", + "expr": "smart_selftest_collection_duration_seconds{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3818,7 +3839,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smartctl_devices", + "expr": "smartctl_devices{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3830,7 +3851,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "smart_selftest_devices", + "expr": "smart_selftest_devices{host=~\"$host\"}", "refId": "B", "instant": false, "range": true, diff --git a/dashboards/zfs-pools.json b/dashboards/zfs-pools.json index 0721bb8..c36a42a 100644 --- a/dashboards/zfs-pools.json +++ b/dashboards/zfs-pools.json @@ -19,6 +19,27 @@ "templating": { "list": [ { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(zfs_pool_health_code, host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "includeAll": false, + "multi": false, + "options": [], + "sort": 1, + "hide": 0, + "regex": "" + }, + { "name": "pool", "label": "Pool", "type": "query", @@ -28,13 +49,17 @@ }, "query": { "qryType": 1, - "query": "label_values(zfs_pool_health_code, pool)", + "query": "label_values(zfs_pool_health_code{host=~\"$host\"}, pool)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "includeAll": true, - "allValue": ".*", "multi": true, + "options": [], + "sort": 1, + "hide": 0, + "regex": "", + "allValue": ".*", "current": { "text": [ "All" @@ -42,9 +67,7 @@ "value": [ "$__all" ] - }, - "options": [], - "sort": 1 + } } ] }, @@ -184,7 +207,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_health_code{pool=~\"$pool\"}", + "expr": "zfs_pool_health_code{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -251,7 +274,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(zfs_pool_data_errors{pool=~\"$pool\"}) or vector(0)", + "expr": "sum(zfs_pool_data_errors{host=~\"$host\", pool=~\"$pool\"}) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -318,7 +341,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(zfs_pool_vdev_errors_total{pool=~\"$pool\"}) or vector(0)", + "expr": "sum(zfs_pool_vdev_errors_total{host=~\"$host\", pool=~\"$pool\"}) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -390,7 +413,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - max(zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"})", + "expr": "time() - max(zfs_pool_last_scrub_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, @@ -469,7 +492,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max(zfs_pool_scrub_in_progress{pool=~\"$pool\"})", + "expr": "max(zfs_pool_scrub_in_progress{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, @@ -481,7 +504,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max(zfs_pool_resilver_in_progress{pool=~\"$pool\"})", + "expr": "max(zfs_pool_resilver_in_progress{host=~\"$host\", pool=~\"$pool\"})", "refId": "B", "instant": true, "range": false, @@ -548,7 +571,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count(zfs_dataset_encrypted{pool=~\"$pool\"} == 1 and zfs_dataset_key_available{pool=~\"$pool\"} == 0) or vector(0)", + "expr": "count(zfs_dataset_encrypted{host=~\"$host\", pool=~\"$pool\"} == 1 and zfs_dataset_key_available{host=~\"$host\", pool=~\"$pool\"} == 0) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -618,7 +641,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_capacity_ratio{pool=~\"$pool\"}", + "expr": "zfs_pool_capacity_ratio{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -688,7 +711,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}", + "expr": "zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -752,7 +775,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -819,7 +842,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count((time() - zfs_snapshot_newest_timestamp_seconds{pool=~\"$pool\"}) > on(label) group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)", + "expr": "count((time() - zfs_snapshot_newest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"}) > on(label) group_left() (2 * zfs_snapshot_interval_seconds{host=~\"$host\"})) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -883,7 +906,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})", + "expr": "time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, @@ -970,7 +993,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_allocated_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_allocated_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -982,7 +1005,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "B", "instant": false, "range": true, @@ -1070,7 +1093,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}", + "expr": "zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -1206,7 +1229,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_size_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -1218,7 +1241,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_allocated_bytes{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_allocated_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "B", "instant": true, "range": false, @@ -1230,7 +1253,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_free_bytes{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "C", "instant": true, "range": false, @@ -1242,7 +1265,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_capacity_ratio{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_capacity_ratio{host=~\"$host\", pool=~\"$pool\"})", "refId": "D", "instant": true, "range": false, @@ -1254,7 +1277,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_fragmentation_ratio{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"})", "refId": "E", "instant": true, "range": false, @@ -1266,7 +1289,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_snapshot_used_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "F", "instant": true, "range": false, @@ -1278,7 +1301,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pool) (zfs_pool_dedup_ratio{pool=~\"$pool\"})", + "expr": "max by (pool) (zfs_pool_dedup_ratio{host=~\"$host\", pool=~\"$pool\"})", "refId": "G", "instant": true, "range": false, @@ -1481,7 +1504,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_vdev_state_code{pool=~\"$pool\"}", + "expr": "zfs_pool_vdev_state_code{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -1589,7 +1612,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_vdev_errors_total{pool=~\"$pool\"}", + "expr": "zfs_pool_vdev_errors_total{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -1737,7 +1760,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"} * 1000", + "expr": "zfs_pool_last_scrub_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"} * 1000", "refId": "A", "instant": true, "range": false, @@ -1749,7 +1772,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_last_scrub_duration_seconds{pool=~\"$pool\"}", + "expr": "zfs_pool_last_scrub_duration_seconds{host=~\"$host\", pool=~\"$pool\"}", "refId": "B", "instant": true, "range": false, @@ -1761,7 +1784,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_last_scrub_errors{pool=~\"$pool\"}", + "expr": "zfs_pool_last_scrub_errors{host=~\"$host\", pool=~\"$pool\"}", "refId": "C", "instant": true, "range": false, @@ -1773,7 +1796,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_last_scrub_repaired_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_last_scrub_repaired_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "D", "instant": true, "range": false, @@ -1785,7 +1808,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_last_resilver_end_timestamp_seconds{pool=~\"$pool\"} * 1000", + "expr": "zfs_pool_last_resilver_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"} * 1000", "refId": "E", "instant": true, "range": false, @@ -1903,7 +1926,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_data_errors{pool=~\"$pool\"}", + "expr": "zfs_pool_data_errors{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -1987,7 +2010,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{pool=~\"$pool\"})", + "expr": "time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, @@ -2059,7 +2082,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "min by (label) (zfs_snapshot_count{pool=~\"$pool\"} / on(label) group_left() zfs_snapshot_retention_target)", + "expr": "min by (label) (zfs_snapshot_count{host=~\"$host\", pool=~\"$pool\"} / on(label) group_left() zfs_snapshot_retention_target{host=~\"$host\"})", "refId": "A", "instant": true, "range": false, @@ -2122,7 +2145,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - min by (label) (zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})", + "expr": "time() - min by (label) (zfs_pool_snapshot_oldest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, @@ -2208,7 +2231,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (label) (zfs_snapshot_count{pool=~\"$pool\"})", + "expr": "sum by (label) (zfs_snapshot_count{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": false, "range": true, @@ -2295,7 +2318,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (label) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})", + "expr": "sum by (label) (zfs_pool_snapshot_used_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "A", "instant": false, "range": true, @@ -2362,7 +2385,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(node_systemd_unit_state{name=~\"zfs-(snapshot|scrub).*\",state=\"failed\"}) or vector(0)", + "expr": "sum(node_systemd_unit_state{host=~\"$host\", name=~\"zfs-(snapshot|scrub).*\", state=\"failed\"}) or vector(0)", "refId": "A", "instant": true, "range": false, @@ -2425,7 +2448,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - node_systemd_timer_last_trigger_seconds{name=~\"zfs-.*\"}", + "expr": "time() - node_systemd_timer_last_trigger_seconds{host=~\"$host\", name=~\"zfs-.*\"}", "refId": "A", "instant": true, "range": false, @@ -2525,7 +2548,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}", + "expr": "zfs_dataset_used_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -2630,7 +2653,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}", + "expr": "zfs_dataset_used_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -2642,7 +2665,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_used_by_dataset_bytes{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_used_by_dataset_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "B", "instant": true, "range": false, @@ -2654,7 +2677,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_used_by_snapshots_bytes{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_used_by_snapshots_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "C", "instant": true, "range": false, @@ -2666,7 +2689,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_referenced_bytes{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_referenced_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "D", "instant": true, "range": false, @@ -2678,7 +2701,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_logical_used_bytes{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_logical_used_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "E", "instant": true, "range": false, @@ -2690,7 +2713,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_available_bytes{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_available_bytes{host=~\"$host\", pool=~\"$pool\"})", "refId": "F", "instant": true, "range": false, @@ -2702,7 +2725,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_compression_ratio{pool=~\"$pool\"})", + "expr": "max by (dataset) (zfs_dataset_compression_ratio{host=~\"$host\", pool=~\"$pool\"})", "refId": "G", "instant": true, "range": false, @@ -2714,7 +2737,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_quota_bytes{pool=~\"$pool\"} > 0)", + "expr": "max by (dataset) (zfs_dataset_quota_bytes{host=~\"$host\", pool=~\"$pool\"} > 0)", "refId": "H", "instant": true, "range": false, @@ -2841,7 +2864,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_compression_ratio{pool=~\"$pool\"} * on(pool, dataset) group_left(compression) zfs_dataset_compression{pool=~\"$pool\"}", + "expr": "zfs_dataset_compression_ratio{host=~\"$host\", pool=~\"$pool\"} * on(pool, dataset) group_left(compression) zfs_dataset_compression{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -2991,7 +3014,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_keystatus_code{pool=~\"$pool\"} * on(pool, dataset) group_left(encryption) zfs_dataset_encryption{pool=~\"$pool\"}", + "expr": "zfs_dataset_keystatus_code{host=~\"$host\", pool=~\"$pool\"} * on(pool, dataset) group_left(encryption) zfs_dataset_encryption{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -3118,7 +3141,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(node_zfs_zpool_dataset_nread{zpool=~\"$pool\"}[$__rate_interval])", + "expr": "rate(node_zfs_zpool_dataset_nread{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", "refId": "A", "instant": false, "range": true, @@ -3130,7 +3153,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "-rate(node_zfs_zpool_dataset_nwritten{zpool=~\"$pool\"}[$__rate_interval])", + "expr": "-rate(node_zfs_zpool_dataset_nwritten{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", "refId": "B", "instant": false, "range": true, @@ -3216,7 +3239,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(node_zfs_zpool_dataset_reads{zpool=~\"$pool\"}[$__rate_interval])", + "expr": "rate(node_zfs_zpool_dataset_reads{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", "refId": "A", "instant": false, "range": true, @@ -3228,7 +3251,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "-rate(node_zfs_zpool_dataset_writes{zpool=~\"$pool\"}[$__rate_interval])", + "expr": "-rate(node_zfs_zpool_dataset_writes{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", "refId": "B", "instant": false, "range": true, @@ -3315,7 +3338,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "node_zfs_arc_size", + "expr": "node_zfs_arc_size{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -3327,7 +3350,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "node_zfs_arc_c_max", + "expr": "node_zfs_arc_c_max{host=~\"$host\"}", "refId": "B", "instant": false, "range": true, @@ -3339,7 +3362,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "node_zfs_arc_c_min", + "expr": "node_zfs_arc_c_min{host=~\"$host\"}", "refId": "C", "instant": false, "range": true, @@ -3426,7 +3449,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min(rate(node_zfs_arc_hits[$__rate_interval]) + rate(node_zfs_arc_misses[$__rate_interval]), 1)", + "expr": "rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval]) / clamp_min(rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval]) + rate(node_zfs_arc_misses{host=~\"$host\"}[$__rate_interval]), 1)", "refId": "A", "instant": false, "range": true, @@ -3513,7 +3536,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(node_zfs_arc_hits[$__rate_interval])", + "expr": "rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval])", "refId": "A", "instant": false, "range": true, @@ -3525,7 +3548,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(node_zfs_arc_misses[$__rate_interval])", + "expr": "rate(node_zfs_arc_misses{host=~\"$host\"}[$__rate_interval])", "refId": "B", "instant": false, "range": true, @@ -3624,7 +3647,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_freeing_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_freeing_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -3636,7 +3659,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_leaked_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_leaked_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "B", "instant": false, "range": true, @@ -3648,7 +3671,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_checkpoint_bytes{pool=~\"$pool\"}", + "expr": "zfs_pool_checkpoint_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "C", "instant": false, "range": true, @@ -3734,7 +3757,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_dedup_ratio{pool=~\"$pool\"}", + "expr": "zfs_pool_dedup_ratio{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -3872,7 +3895,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_pool_health_code{pool=~\"$pool\"}", + "expr": "zfs_pool_health_code{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, @@ -4005,7 +4028,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_recordsize_bytes{pool=~\"$pool\"}", + "expr": "zfs_dataset_recordsize_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, @@ -4017,7 +4040,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_quota_bytes{pool=~\"$pool\"}", + "expr": "zfs_dataset_quota_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "B", "instant": true, "range": false, @@ -4029,7 +4052,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_refquota_bytes{pool=~\"$pool\"}", + "expr": "zfs_dataset_refquota_bytes{host=~\"$host\", pool=~\"$pool\"}", "refId": "C", "instant": true, "range": false, @@ -4041,7 +4064,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_dataset_autosnapshot_enabled{pool=~\"$pool\"}", + "expr": "zfs_dataset_autosnapshot_enabled{host=~\"$host\", pool=~\"$pool\"}", "refId": "D", "instant": true, "range": false, @@ -4173,7 +4196,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "zfs_metrics_collection_duration_seconds", + "expr": "zfs_metrics_collection_duration_seconds{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -4260,7 +4283,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "time() - node_textfile_mtime_seconds", + "expr": "time() - node_textfile_mtime_seconds{host=~\"$host\"}", "refId": "A", "instant": false, "range": true, @@ -4358,7 +4381,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "node_textfile_scrape_error", + "expr": "node_textfile_scrape_error{host=~\"$host\"}", "refId": "B", "instant": false, "range": true, diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..c508296 --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1787736819, + "narHash": "sha256-cV5xEJJK3BvhU8rEd4mC9UsmDi5qscv/kzGPhBRC5WA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9fbb54b33e91ee4ca368e35a78e0613c720600b3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..1a9c661 --- /dev/null +++ b/flake.nix @@ -0,0 +1,109 @@ +{ + description = "Host telemetry collectors and a Prometheus/Loki/Grafana server, as two separable NixOS modules"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + outputs = { self, nixpkgs }: + let + systems = [ "x86_64-linux" "aarch64-linux" ]; + forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system}); + in + { + # Paths rather than functions, so the modules are evaluated with the *consumer's* + # nixpkgs. This flake's own nixpkgs input therefore only affects packages, checks + # and devShells -- importing these modules never pulls in a second nixpkgs. + nixosModules = { + collectors = ./modules/collectors.nix; + server = ./modules/server.nix; + default = ./modules; + }; + + # The textfile collectors, runnable by hand. Both take --output PATH and write + # Prometheus exposition format; running them against a scratch file is how you + # check what a machine actually reports. + packages = forAllSystems (pkgs: + # callPackages, not callPackage: the file returns an attrset of + # derivations, and callPackage would wrap the whole set in + # override/overrideDerivation attrs that are not derivations. + let collectors = pkgs.callPackages ./modules/packages.nix { }; + in collectors // { default = collectors.zfs-metrics; } + ); + + checks = forAllSystems (pkgs: { + # promtool over the recording rules. The server module also runs this at build + # time via services.prometheus.checkConfig, but that only fires for people who + # actually deploy it. + prometheus-rules = pkgs.runCommand "check-prometheus-rules" + { nativeBuildInputs = [ pkgs.prometheus.cli ]; } + '' + promtool check rules ${./rules}/*.yml + touch "$out" + ''; + + # The dashboard JSON is generated by the scripts next to it and committed, so it + # can rot silently if someone edits a generator without re-running it. Regenerate + # here and diff. + dashboards-current = pkgs.runCommand "check-dashboards-current" + { nativeBuildInputs = [ pkgs.python3 pkgs.diffutils ]; } + '' + cp -r ${./dashboards} work + chmod -R u+w work + cd work + + # Note the loop variable is not called "out": that is the derivation's own + # output path, and shadowing it makes the final touch silently write + # somewhere else and the build fail with "failed to produce output path". + check() { + local generator="$1" committed="$2" + python3 "$generator" > regenerated.json + if ! diff -u "$committed" regenerated.json; then + echo + echo "dashboards/$committed is stale." + echo "Re-run: python3 dashboards/$generator > dashboards/$committed" + exit 1 + fi + } + + check gen_smart.py smart.json + check gen_zfs.py zfs-pools.json + touch "$out" + ''; + + # Both halves have to evaluate, separately and together. Catches option typos + # and missing mkIf guards without needing a real machine. + modules-eval = (nixpkgs.lib.nixosSystem { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + self.nixosModules.default + ({ lib, ... }: { + services.telemetry.collectors.enable = true; + services.telemetry.server = { + enable = true; + collectorHosts = [ "one" "two" ]; + proxy.enable = true; + # Never read during evaluation; only has to satisfy the assertion. + grafana.secretKeyFile = "/run/secrets/grafana-secret-key"; + }; + boot.loader.grub.devices = [ "/dev/sda" ]; + fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; }; + system.stateVersion = "25.11"; + }) + ]; + }).config.system.build.toplevel; + }); + + devShells = forAllSystems (pkgs: { + default = pkgs.mkShell { + packages = [ + pkgs.python3 # the dashboard generators and collectors + pkgs.prometheus.cli # promtool, for the recording rules + pkgs.prometheus # a real Prometheus to run dashboard queries against + pkgs.jq + pkgs.smartmontools + ]; + }; + }); + + formatter = forAllSystems (pkgs: pkgs.nixpkgs-fmt); + }; +} diff --git a/lgtm.nix b/lgtm.nix deleted file mode 100644 index a4144ec..0000000 --- a/lgtm.nix +++ /dev/null @@ -1,534 +0,0 @@ -{ pkgs, config, lib, ... }: - -let - # The hostname the container uses to reach the host's exporters. - hostName = "poweredge"; - - # One scrape interval to rule them all. This MUST match the Grafana Prometheus - # datasource's `timeInterval` below, otherwise Grafana computes $__rate_interval - # from the wrong assumption and every rate()-based panel intermittently renders - # "No data". (Prometheus' own default is 1m, which is what broke the Node Exporter - # Full dashboard: $__rate_interval resolved to 60s, so rate(x[60s]) over 60s-spaced - # samples usually saw fewer than the two samples rate() requires.) - scrapeInterval = "15s"; - - # Where the host's zpool collector drops its metrics for node_exporter to pick up. - textfileDir = "/var/lib/prometheus-node-exporter-textfile"; - - ports = { - node = 9100; - smartctl = 9633; - loki = 3100; - prometheus = 9090; - grafana = 3000; - }; - - # Stable datasource UIDs, referenced by the dashboards below. - dsUid = { prometheus = "prometheus"; loki = "loki"; }; - - ########################################################################## - # Dashboard plumbing - ########################################################################## - - # Dashboards exported from grafana.com carry an `__inputs` block and reference - # their datasource as "${DS_PROMETHEUS}". That placeholder is only substituted by - # Grafana's interactive *import* flow — file-based provisioning never runs it, so - # the uid stays literal and every panel fails to resolve a datasource. This filter - # performs the substitution at build time, using our fixed datasource UIDs. - dashboardFixup = pkgs.writeText "grafana-dashboard-fixup.jq" '' - def dsuid($pluginId): - if $pluginId == "prometheus" then $prom - elif $pluginId == "loki" then $loki - else "" end; - - def subst($name; $uid): - walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end); - - reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId))) - | del(.__inputs, .__requires) - # Provisioned dashboards must not carry an id, or Grafana rejects them on - # collision with whatever already occupies that id in its database. - | .id = null - ''; - - fixupDashboard = name: src: pkgs.runCommand "grafana-dashboard-${name}.json" - { nativeBuildInputs = [ pkgs.jq ]; } - '' - jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \ - --arg loki ${lib.escapeShellArg dsUid.loki} \ - -f ${dashboardFixup} < ${src} > "$out" - ''; - - # grafana.com dashboard, pinned by revision. - mkGrafanaComDashboard = { name, id, rev, sha256 }: - fixupDashboard name (pkgs.fetchurl { - url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download"; - inherit sha256; - }); - - dashboards = { - # https://grafana.com/grafana/dashboards/1860 — Node Exporter Full - node-exporter-full = mkGrafanaComDashboard { - name = "node-exporter-full"; id = 1860; rev = 45; - sha256 = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY="; - }; - - # https://grafana.com/grafana/dashboards/22604 — Smartctl Exporter - smartctl-exporter = mkGrafanaComDashboard { - name = "smartctl-exporter"; id = 22604; rev = 2; - sha256 = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88="; - }; - - # Dashboard 15362 ("ZFS Pool Metrics") used to live here. It is an *InfluxDB* - # dashboard — its queries are InfluxQL against a `zpool_stats` measurement — so - # it can never render against Prometheus, no matter how it is provisioned. This - # is a Prometheus-native replacement built on the metrics we actually export: - # node_exporter's zfs collector plus the zpool textfile collector below. - zfs-pools = fixupDashboard "zfs-pools" ./dashboards/zfs-pools.json; - - # CrystalDiskInfo-style per-drive SMART view. Generated by - # ./dashboards/gen_smart.py; the Good/Caution/Bad verdict it displays comes - # from the recording rules in ./rules/smartctl.yml. - smart = fixupDashboard "smart" ./dashboards/smart.json; - }; - - ########################################################################## - # ZFS metrics (host) - ########################################################################## - - # node_exporter's zfs collector reads /proc/spl/kstat/zfs, which gives us ARC - # statistics, per-dataset objset counters and pool state -- but nothing about - # capacity, fragmentation, vdev errors, scrub history, compression, encryption or - # snapshots. OpenZFS 2.2+ also no longer publishes the per-pool `io` kstat, so - # node_zfs_zpool_* is largely empty. Everything else comes from the zpool/zfs CLIs - # via ./collectors/zfs-metrics.py, scraped through the textfile collector. - # - # The wrapper exists only to put the matching zfs userland on PATH; the collector - # itself is a plain script so it can be run and tested by hand. - snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ]; - - retentionArgs = lib.optionals config.services.zfs.autoSnapshot.enable ( - lib.concatMap (label: [ - "--retention" - "${label}=${toString config.services.zfs.autoSnapshot.${label}}" - ]) snapshotLabels - ); - - zfsCollector = pkgs.writeShellApplication { - name = "zfs-textfile-collector"; - runtimeInputs = [ config.boot.zfs.package ]; - text = '' - exec ${pkgs.python3}/bin/python3 ${./collectors/zfs-metrics.py} \ - --output ${textfileDir}/zfs.prom ${lib.escapeShellArgs retentionArgs} - ''; - }; - - # smartctl_exporter reads only the entry *count* out of each drive's self-test log - # and discards the log table, so it cannot say when a drive last ran a short or - # extended test, or whether it passed. This parses `smartctl -j -l selftest` - # directly. Device labels are bare names ("sda") so they join against the - # exporter's own series. - smartSelftestCollector = pkgs.writeShellApplication { - name = "smart-selftest-collector"; - runtimeInputs = [ pkgs.smartmontools ]; - text = '' - exec ${pkgs.python3}/bin/python3 ${./collectors/smart-selftest.py} \ - --output ${textfileDir}/smart-selftest.prom - ''; - }; -in -{ - ########################################################################## - # Host: log shipping - ########################################################################## - - services.alloy.enable = true; - - environment.etc."alloy/config.alloy".text = '' - loki.source.journal "journal" { - forward_to = [loki.write.default.receiver] - relabel_rules = loki.relabel.journal.rules - max_age = "12h" - } - - loki.relabel "journal" { - forward_to = [] - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "unit" - } - rule { - source_labels = ["__journal__hostname"] - target_label = "host" - } - rule { - source_labels = ["__journal_priority_keyword"] - target_label = "level" - } - } - - loki.write "default" { - endpoint { - url = "http://lgtm:${toString ports.loki}/loki/api/v1/push" - } - } - ''; - - ########################################################################## - # Host: telemetry sources - # - # These have to live on the host, not in the container: they read host - # hardware and kernel state (/proc, /sys, SMART, ZFS ioctls) that a - # container has no view of. Only the web services (Grafana, Prometheus, - # Loki) run in the container. - ########################################################################## - - services.prometheus.exporters.node = { - enable = true; - port = ports.node; - listenAddress = "0.0.0.0"; - - # NOTE: enabledCollectors *adds* to node_exporter's defaults — cpu, diskstats, - # filesystem, meminfo, zfs, hwmon, pressure and friends are already on, so - # listing them here did nothing. Only genuinely opt-in collectors belong here. - enabledCollectors = [ - "systemd" # node_systemd_* — the "Systemd" row - "processes" # node_processes_* — "Processes Detailed States", "PIDs", "Threads" - # "interrupts" # fills the "IRQ Detail" panel, but emits one series per - # # (cpu, irq) pair — thousands of series on a many-core box. - # # Enable only if you want that one panel badly enough. - ]; - - extraFlags = [ - "--collector.textfile.directory=${textfileDir}" - # node_systemd_unit_start_time_seconds, so the dashboards can show when the - # zfs-snapshot-*/zfs-scrub units last ran rather than only whether they failed. - "--collector.systemd.enable-start-time-metrics" - ]; - }; - - # Upstream hardens the unit with ProtectHome=true, which makes /home look empty and - # blanks out the filesystem panels for any dataset mounted under it. read-only still - # allows statfs while keeping the sandbox. - systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only"; - - services.prometheus.exporters.smartctl = { - enable = true; - port = ports.smartctl; - listenAddress = "0.0.0.0"; - # Disks are direct-attached (AHCI/HBA), so auto-discovery via `smartctl --scan` - # finds them; no explicit `devices` list needed. Throttle how often we actually - # shell out to smartctl — SMART data changes slowly and polling spins up disks. - maxInterval = "60s"; - }; - - # SMART self-test history. Reading the log does not spin an idle disk up, but the - # data only changes when a test finishes, so this runs far less often than the - # exporter scrapes. - systemd.services.smart-selftest-collector = { - description = "Collect SMART self-test history for the Prometheus node exporter"; - serviceConfig = { - Type = "oneshot"; - ExecStart = lib.getExe smartSelftestCollector; - User = "root"; - StateDirectory = "prometheus-node-exporter-textfile"; - StateDirectoryMode = "0755"; - UMask = "0022"; - TimeoutStartSec = "180s"; - - # Same shape as the smartctl exporter's own hardening: it needs raw device - # access to talk to the disks, and nothing else. - AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ]; - CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ]; - DevicePolicy = "closed"; - DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ]; - LockPersonality = true; - MemoryDenyWriteExecute = true; - NoNewPrivileges = true; - PrivateNetwork = true; - PrivateTmp = true; - ProtectControlGroups = true; - ProtectHome = true; - ProtectHostname = true; - ProtectKernelLogs = true; - ProtectKernelModules = true; - ProtectSystem = "strict"; - RestrictAddressFamilies = [ "AF_UNIX" ]; - RestrictNamespaces = true; - RestrictRealtime = true; - RestrictSUIDSGID = true; - SystemCallArchitectures = "native"; - }; - }; - - systemd.timers.smart-selftest-collector = { - description = "Collect SMART self-test history periodically"; - wantedBy = [ "timers.target" ]; - timerConfig = { - OnBootSec = "3min"; - OnUnitActiveSec = "15min"; - AccuracySec = "1min"; - Unit = "smart-selftest-collector.service"; - }; - }; - - # Pool capacity and health, vdev errors, scrub history, dataset properties and - # snapshot retention. See zfsCollector above for why node_exporter's built-in zfs - # collector isn't enough on its own. - systemd.services.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled { - description = "Collect ZFS metrics for the Prometheus node exporter"; - serviceConfig = { - Type = "oneshot"; - ExecStart = lib.getExe zfsCollector; - User = "root"; - StateDirectory = "prometheus-node-exporter-textfile"; - StateDirectoryMode = "0755"; - UMask = "0022"; # node_exporter runs as a different user and must read these - # Listing every snapshot on a busy pool is the slow part; keep this well - # above the subprocess timeout inside the collector. - TimeoutStartSec = "180s"; - - # Hardening — this only needs to read ZFS state and write one file. - # CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated - # on it; everything else is dropped. - CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ]; - LockPersonality = true; - MemoryDenyWriteExecute = true; - NoNewPrivileges = true; - PrivateNetwork = true; - PrivateTmp = true; - ProtectControlGroups = true; - ProtectHome = true; - ProtectHostname = true; - ProtectKernelLogs = true; - ProtectKernelModules = true; - ProtectSystem = "strict"; - RestrictAddressFamilies = [ "AF_UNIX" ]; - RestrictNamespaces = true; - RestrictRealtime = true; - RestrictSUIDSGID = true; - SystemCallArchitectures = "native"; - SystemCallFilter = [ "@system-service" ]; - }; - }; - - systemd.timers.zfs-textfile-collector = lib.mkIf config.boot.zfs.enabled { - description = "Collect ZFS metrics periodically"; - wantedBy = [ "timers.target" ]; - timerConfig = { - OnBootSec = "1min"; - OnUnitActiveSec = "1min"; - AccuracySec = "5s"; - Unit = "zfs-textfile-collector.service"; - }; - }; - - # The collector used to write zpool.prom. Nothing removes a stale textfile once its - # writer is gone, and node_exporter would happily keep serving those frozen metrics - # forever, so delete it on activation. - systemd.tmpfiles.rules = lib.mkIf config.boot.zfs.enabled [ - "r ${textfileDir}/zpool.prom" - ]; - - # Reachable from the container, which sits on br-lan0 with its own address. - # (Port 9273 is gone — that was telegraf/zpool_influxdb, replaced by the textfile - # collector above, which needs no listener of its own.) - networking.firewall.allowedTCPPorts = [ ports.node ports.smartctl ]; - - ########################################################################## - # Secrets - ########################################################################## - - sops.secrets.grafana-secret-key = { - sopsFile = ./resources/secrets/grafana.yaml; - key = "secret_key"; - }; - - ########################################################################## - # Container: Grafana / Prometheus / Loki - ########################################################################## - - containers.grafana = { - autoStart = true; - privateNetwork = true; - hostBridge = "br-lan0"; - localMacAddress = "02:00:00:00:00:06"; - - bindMounts."/run/secrets/grafana-secret_key" = { - hostPath = config.sops.secrets.grafana-secret-key.path; - isReadOnly = true; - }; - - config = { lib, pkgs, config, ... }: { - # Network - networking.enableIPv6 = false; - networking.interfaces.eth0.useDHCP = true; - networking.firewall.allowedTCPPorts = [ - 80 # Caddy -> Grafana - ports.loki # host's Alloy pushes here - ports.prometheus # handy for debugging scrape health directly - ]; - - # --------------------------------------------------------------- - # Loki - # --------------------------------------------------------------- - services.loki = { - enable = true; - configuration = { - server = { - http_listen_port = ports.loki; - # Explicit, since the host's Alloy needs to reach it to push logs. - http_listen_address = "0.0.0.0"; - }; - auth_enabled = false; - - common = { - path_prefix = "/var/lib/loki"; - storage.filesystem = { - chunks_directory = "/var/lib/loki/chunks"; - rules_directory = "/var/lib/loki/rules"; - }; - ring = { - instance_addr = "127.0.0.1"; - kvstore.store = "inmemory"; - }; - replication_factor = 1; - }; - - schema_config.configs = [{ - from = "2024-01-01"; - store = "tsdb"; - object_store = "filesystem"; - schema = "v13"; - index = { prefix = "index_"; period = "24h"; }; - }]; - }; - }; - - # --------------------------------------------------------------- - # Prometheus - # --------------------------------------------------------------- - services.prometheus = { - enable = true; - - # Without this Prometheus falls back to its built-in 1m scrape interval, - # which desynchronises from what Grafana assumes and breaks every - # rate()-based panel. See the comment on `scrapeInterval` at the top. - globalConfig = { - scrape_interval = scrapeInterval; - evaluation_interval = scrapeInterval; - scrape_timeout = "10s"; - }; - - # Drive health verdict. Kept as recording rules rather than inlined in - # dashboard panels so the logic is written once and reviewable on its - # own. services.prometheus.checkConfig runs promtool at build time, so - # a broken rule fails the build instead of the service. - ruleFiles = [ ./rules/smartctl.yml ]; - - scrapeConfigs = [ - { - job_name = "node"; - static_configs = [{ targets = [ "${hostName}:${toString ports.node}" ]; }]; - } - { - job_name = "smartctl"; - # Querying SMART is comparatively slow and the data barely moves, so - # this job opts out of the global 15s. Safe to do here because the - # smartctl dashboard uses no rate() queries at all. - scrape_interval = "60s"; - scrape_timeout = "30s"; - static_configs = [{ targets = [ "${hostName}:${toString ports.smartctl}" ]; }]; - } - ]; - }; - - # --------------------------------------------------------------- - # Grafana - # https://wiki.nixos.org/wiki/Grafana - # --------------------------------------------------------------- - services.grafana = { - enable = true; - settings.server = { http_addr = "0.0.0.0"; http_port = ports.grafana; }; - - provision = { - enable = true; - - # Mutable dashboard provider pulling from /etc/grafana-dashboards, so - # dashboards can be declared with environment.etc below. - dashboards.settings.providers = [{ - name = "my dashboards"; - disableDeletion = true; - options = { - path = "/etc/grafana-dashboards"; - foldersFromFilesStructure = true; - }; - }]; - - # Grafana cannot change the UID of an existing datasource in place. Its - # provisioner finds the datasource by name, then calls UpdateDataSource, - # which re-fetches using `id AND uid` — with a new uid that row doesn't - # match, so it fails with "data source not found" and Grafana refuses to - # start. Deleting first turns the upsert into a plain insert; the - # provisioner runs deletes before inserts for exactly this case, and - # suppresses the deletion event because the datasource is about to be - # re-created. Safe to leave in permanently: both datasources are fully - # declarative, and dashboards bind to them by UID, not by database ID. - datasources.settings.deleteDatasources = [ - { name = "Prometheus"; orgId = 1; } - { name = "Loki"; orgId = 1; } - ]; - - datasources.settings.datasources = [ - { - name = "Prometheus"; - type = "prometheus"; - uid = dsUid.prometheus; # fixed, so dashboards can reference it - url = "http://127.0.0.1:${toString ports.prometheus}"; - isDefault = true; - jsonData = { - # Tells Grafana how often the data is actually scraped. This is - # what $__rate_interval is derived from; leaving it unset makes - # Grafana guess 15s and mis-size every rate() window. - timeInterval = scrapeInterval; - httpMethod = "POST"; - }; - } - { - name = "Loki"; - type = "loki"; - uid = dsUid.loki; - url = "http://127.0.0.1:${toString ports.loki}"; - } - ]; - }; - }; - - # Grafana secret key - systemd.services.grafana.serviceConfig.LoadCredential = [ - "secret_key:/run/secrets/grafana-secret_key" - ]; - services.grafana.settings.security.secret_key = - "$__file{/run/credentials/grafana.service/secret_key}"; - - # --------------------------------------------------------------- - # Dashboards - # --------------------------------------------------------------- - environment.etc = lib.mapAttrs' (name: drv: - lib.nameValuePair "grafana-dashboards/${name}.json" { source = drv; } - ) dashboards; - - # --------------------------------------------------------------- - # Reverse proxy - # --------------------------------------------------------------- - services.caddy = { - enable = true; - virtualHosts.":80".extraConfig = '' - reverse_proxy localhost:${toString ports.grafana} - ''; - }; - - system.stateVersion = "26.05"; - }; - }; -} diff --git a/modules/collectors.nix b/modules/collectors.nix new file mode 100644 index 0000000..6f328ee --- /dev/null +++ b/modules/collectors.nix @@ -0,0 +1,368 @@ +# Host-side telemetry sources. +# +# Everything here has to run on the host, not in a container: it reads host hardware and +# kernel state -- /proc, /sys, SMART over raw block devices, ZFS ioctls -- that a +# container has no view of. The half that can be containerised is modules/server.nix. +{ config, lib, pkgs, ... }: + +let + inherit (lib) mkIf mkOption mkEnableOption mkForce types optionals concatMap; + + cfg = config.services.telemetry.collectors; + ports = config.services.telemetry.ports; + + collectorPkgs = pkgs.callPackages ./packages.nix { + # Match the userland to the running kernel module. + zfs = config.boot.zfs.package; + }; + + # services.zfs.autoSnapshot's per-label keep counts, handed to the collector so it can + # report how far each label is from its target rather than just how many snapshots + # exist. Silently empty when auto-snapshotting is off, which the collector handles. + snapshotLabels = [ "frequent" "hourly" "daily" "weekly" "monthly" ]; + + retentionArgs = optionals config.services.zfs.autoSnapshot.enable ( + concatMap + (label: [ + "--retention" + "${label}=${toString config.services.zfs.autoSnapshot.${label}}" + ]) + snapshotLabels + ); + + # Hardening common to both collectors. Each writes exactly one file into the textfile + # directory and talks to nothing over the network. + hardening = { + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateNetwork = true; + PrivateTmp = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectSystem = "strict"; + RestrictAddressFamilies = [ "AF_UNIX" ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + }; + + mkCollectorService = { description, execStart, extraServiceConfig ? { } }: { + inherit description; + serviceConfig = hardening // { + Type = "oneshot"; + ExecStart = execStart; + User = "root"; + StateDirectory = "prometheus-node-exporter-textfile"; + StateDirectoryMode = "0755"; + # node_exporter runs as a different user and has to be able to read what we write. + UMask = "0022"; + TimeoutStartSec = "180s"; + } // extraServiceConfig; + }; + + mkCollectorTimer = { description, onBootSec, onUnitActiveSec, accuracySec, unit }: { + inherit description; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = onBootSec; + OnUnitActiveSec = onUnitActiveSec; + AccuracySec = accuracySec; + Unit = unit; + }; + }; +in +{ + imports = [ ./common.nix ]; + + options.services.telemetry.collectors = { + enable = mkEnableOption "host-side telemetry collectors (node exporter, SMART, ZFS)"; + + listenAddress = mkOption { + type = types.str; + default = "0.0.0.0"; + description = '' + Address the exporters bind to. The default is deliberately permissive because + the whole point of splitting the halves is that the server scrapes these from + somewhere else -- a container on a bridge, or another machine. Narrow it if the + server runs on this host. + ''; + }; + + openFirewall = mkOption { + type = types.bool; + default = true; + description = '' + Open the node and smartctl exporter ports. Needed whenever the server is not on + this host's loopback, which includes a container with `privateNetwork`. + ''; + }; + + textfileDir = mkOption { + type = types.path; + default = "/var/lib/prometheus-node-exporter-textfile"; + description = '' + Directory the ZFS and SMART collectors write `.prom` files into, and that + node_exporter's textfile collector reads. Changing this also changes the + StateDirectory of both collector units. + ''; + }; + + node = { + enable = mkOption { + type = types.bool; + default = true; + description = "Run the Prometheus node exporter."; + }; + + extraCollectors = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "interrupts" ]; + description = '' + Additional node_exporter collectors, on top of the systemd, processes and + textfile collectors this module enables. + + Note this *adds* to node_exporter's defaults -- cpu, diskstats, filesystem, + meminfo, zfs, hwmon and pressure are already on, so listing those does + nothing. `interrupts` is the notable opt-in: it fills the Node Exporter Full + "IRQ Detail" panel but emits one series per (cpu, irq) pair, which is + thousands of series on a many-core box. + ''; + }; + }; + + smart = { + enable = mkOption { + type = types.bool; + default = true; + description = '' + Run the smartctl exporter and the self-test history collector. + ''; + }; + + devices = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "/dev/sda" "/dev/nvme0" ]; + description = '' + Devices to poll. Empty means auto-discovery via `smartctl --scan`, which is + correct for direct-attached AHCI/HBA disks. Behind a RAID controller you + generally have to list devices explicitly with their `-d` type. + ''; + }; + + maxInterval = mkOption { + type = types.str; + default = "60s"; + description = '' + Floor on how often the exporter actually shells out to smartctl, independent + of how often Prometheus scrapes it. SMART data moves slowly and polling can + spin up idle disks. + ''; + }; + + selfTestInterval = mkOption { + type = types.str; + default = "15min"; + description = '' + How often to re-read the self-test logs. The data only changes when a test + finishes, so this can be far slower than the exporter's own polling. + ''; + }; + }; + + zfs = { + enable = mkOption { + type = types.bool; + default = config.boot.zfs.enabled; + defaultText = lib.literalExpression "config.boot.zfs.enabled"; + description = '' + Collect pool capacity and health, vdev errors, scrub history, dataset + properties and snapshot retention from the zpool/zfs CLIs. + ''; + }; + + interval = mkOption { + type = types.str; + default = "1min"; + description = '' + How often to re-read pool and dataset state. Listing every snapshot on a busy + pool is the slow part; if this starts overlapping its own runs, raise it. + ''; + }; + }; + + logs = { + enable = mkOption { + type = types.bool; + default = true; + description = "Ship the systemd journal to Loki with Grafana Alloy."; + }; + + lokiUrl = mkOption { + type = types.str; + default = "http://localhost:${toString ports.loki}"; + defaultText = lib.literalExpression ''"http://localhost:''${toString config.services.telemetry.ports.loki}"''; + example = "http://metrics.lan:3100"; + description = '' + Base URL of the Loki instance to push to. Point this at wherever + modules/server.nix runs; the `/loki/api/v1/push` path is appended. + ''; + }; + + maxAge = mkOption { + type = types.str; + default = "12h"; + description = '' + How far back into the journal to read on startup. Anything older than this is + skipped rather than replayed into Loki. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + # ------------------------------------------------------------------------------ + # node exporter + # ------------------------------------------------------------------------------ + services.prometheus.exporters.node = mkIf cfg.node.enable { + enable = true; + port = ports.node; + inherit (cfg) listenAddress; + + enabledCollectors = [ + "systemd" # node_systemd_* — the "Systemd" row, and the zfs-* timer panels + "processes" # node_processes_* — "Processes Detailed States", "PIDs", "Threads" + ] ++ cfg.node.extraCollectors; + + extraFlags = [ + "--collector.textfile.directory=${cfg.textfileDir}" + # node_systemd_unit_start_time_seconds, so the dashboards can show when the + # zfs-snapshot-* and zfs-scrub units last ran rather than only whether they + # failed. + "--collector.systemd.enable-start-time-metrics" + ]; + }; + + # Upstream hardens the unit with ProtectHome=true, which makes /home look empty and + # blanks out the filesystem panels for any dataset mounted under it. read-only still + # allows statfs while keeping the sandbox. + systemd.services.prometheus-node-exporter = mkIf cfg.node.enable { + serviceConfig.ProtectHome = mkForce "read-only"; + }; + + # ------------------------------------------------------------------------------ + # SMART + # ------------------------------------------------------------------------------ + services.prometheus.exporters.smartctl = mkIf cfg.smart.enable { + enable = true; + port = ports.smartctl; + inherit (cfg) listenAddress; + inherit (cfg.smart) devices maxInterval; + }; + + systemd.services.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorService { + description = "Collect SMART self-test history and drive identity"; + execStart = "${lib.getExe collectorPkgs.smart-selftest} --output ${cfg.textfileDir}/smart-selftest.prom"; + # Same shape as the smartctl exporter's own hardening: raw device access to talk + # to the disks, and nothing else. + extraServiceConfig = { + AmbientCapabilities = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ]; + CapabilityBoundingSet = [ "CAP_SYS_RAWIO" "CAP_SYS_ADMIN" ]; + DevicePolicy = "closed"; + DeviceAllow = [ "block-blkext rw" "block-sd rw" "char-nvme rw" ]; + }; + }); + + systemd.timers.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorTimer { + description = "Collect SMART self-test history periodically"; + onBootSec = "3min"; + onUnitActiveSec = cfg.smart.selfTestInterval; + accuracySec = "1min"; + unit = "smart-selftest-collector.service"; + }); + + # ------------------------------------------------------------------------------ + # ZFS + # ------------------------------------------------------------------------------ + systemd.services.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorService { + description = "Collect ZFS pool, dataset and snapshot metrics"; + execStart = lib.escapeShellArgs ([ + (lib.getExe collectorPkgs.zfs-metrics) + "--output" + "${cfg.textfileDir}/zfs.prom" + ] ++ retentionArgs); + # CAP_SYS_ADMIN stays in the bounding set because some zfs ioctls are gated on it; + # everything else is dropped. + extraServiceConfig = { + CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_OVERRIDE" ]; + SystemCallFilter = [ "@system-service" ]; + }; + }); + + systemd.timers.zfs-textfile-collector = mkIf cfg.zfs.enable (mkCollectorTimer { + description = "Collect ZFS metrics periodically"; + onBootSec = "1min"; + onUnitActiveSec = cfg.zfs.interval; + accuracySec = "5s"; + unit = "zfs-textfile-collector.service"; + }); + + # An earlier version of the ZFS collector wrote zpool.prom. Nothing removes a stale + # textfile once its writer is gone, and node_exporter will happily keep serving + # those frozen metrics forever, so delete it on activation. + systemd.tmpfiles.rules = mkIf cfg.zfs.enable [ + "r ${cfg.textfileDir}/zpool.prom" + ]; + + # ------------------------------------------------------------------------------ + # Journal shipping + # ------------------------------------------------------------------------------ + services.alloy.enable = mkIf cfg.logs.enable true; + + environment.etc."alloy/config.alloy" = mkIf cfg.logs.enable { + text = '' + loki.source.journal "journal" { + forward_to = [loki.write.default.receiver] + relabel_rules = loki.relabel.journal.rules + max_age = "${cfg.logs.maxAge}" + } + + loki.relabel "journal" { + forward_to = [] + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal__hostname"] + target_label = "host" + } + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + } + + loki.write "default" { + endpoint { + url = "${cfg.logs.lokiUrl}/loki/api/v1/push" + } + } + ''; + }; + + # ------------------------------------------------------------------------------ + # Firewall + # ------------------------------------------------------------------------------ + networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall ( + optionals cfg.node.enable [ ports.node ] + ++ optionals cfg.smart.enable [ ports.smartctl ] + ); + }; +} diff --git a/modules/common.nix b/modules/common.nix new file mode 100644 index 0000000..480aa62 --- /dev/null +++ b/modules/common.nix @@ -0,0 +1,46 @@ +# Options both halves need to agree on. +# +# The collectors bind these ports; the server scrapes them and pushes to them. When the +# two halves live in separate NixOS configurations (host + container) there is no shared +# `config` to read, so each side sets its own copy and it is on you to keep them equal — +# which is why they are gathered in one small file rather than scattered. +# +# Imported by both modules/collectors.nix and modules/server.nix. Importing the same +# path twice is harmless: the module system deduplicates imports by path, so a +# configuration that uses both halves does not end up declaring these twice. +{ lib, ... }: + +let + inherit (lib) mkOption types; + + portOption = default: description: mkOption { + inherit default description; + type = types.port; + }; +in +{ + options.services.telemetry.ports = { + node = portOption 9100 '' + Port the Prometheus node exporter listens on. Also where the server's `node` + scrape job points. + ''; + + smartctl = portOption 9633 '' + Port the smartctl exporter listens on. Also where the server's `smartctl` scrape + job points. + ''; + + loki = portOption 3100 '' + Port Loki listens on. Also where the collectors' log shipper pushes to. + ''; + + prometheus = portOption 9090 '' + Port Prometheus listens on. + ''; + + grafana = portOption 3000 '' + Port Grafana listens on. If the reverse proxy is enabled this stays internal and + the proxy is what you connect to. + ''; + }; +} diff --git a/modules/dashboards.nix b/modules/dashboards.nix new file mode 100644 index 0000000..125e3e2 --- /dev/null +++ b/modules/dashboards.nix @@ -0,0 +1,75 @@ +# Grafana dashboards, built into a form file-provisioning can actually use. +# +# Returns an attrset of { name = derivation; }, where each derivation is a single +# dashboard JSON file. modules/server.nix drops these into /etc/grafana-dashboards. +{ lib, runCommand, fetchurl, jq, dsUid }: + +let + # Dashboards exported from grafana.com carry an `__inputs` block and reference their + # datasource as "''${DS_PROMETHEUS}". That placeholder is only substituted by Grafana's + # interactive *import* flow -- file provisioning never runs it, so the uid stays + # literal and every panel fails to resolve a datasource. Substitute at build time. + # + # A dashboard-level `datasource`-type template variable (lowercase, e.g. + # "''${ds_prometheus}") is a different mechanism and resolves fine on its own; this + # only touches names declared in __inputs. + fixupProgram = '' + def dsuid($pluginId): + if $pluginId == "prometheus" then $prom + elif $pluginId == "loki" then $loki + else "" end; + + def subst($name; $uid): + walk(if type == "string" then gsub("\\$\\{" + $name + "\\}"; $uid) else . end); + + reduce (.__inputs // [])[] as $i (.; subst($i.name; dsuid($i.pluginId))) + | del(.__inputs, .__requires) + # Provisioned dashboards must not carry an id, or Grafana rejects them on collision + # with whatever already occupies that id in its database. + | .id = null + ''; + + fixup = name: src: runCommand "grafana-dashboard-${name}.json" + { nativeBuildInputs = [ jq ]; } + '' + jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \ + --arg loki ${lib.escapeShellArg dsUid.loki} \ + ${lib.escapeShellArg fixupProgram} < ${src} > "$out" + ''; + + fromGrafanaCom = { name, id, rev, hash }: + fixup name (fetchurl { + url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download"; + inherit hash; + }); +in +{ + # https://grafana.com/grafana/dashboards/1860 + node-exporter-full = fromGrafanaCom { + name = "node-exporter-full"; + id = 1860; + rev = 45; + hash = "sha256-GExrdAnzBtp1Ul13cvcZRbEM6iOtFrXXjEaY6g6lGYY="; + }; + + # https://grafana.com/grafana/dashboards/22604 + smartctl-exporter = fromGrafanaCom { + name = "smartctl-exporter"; + id = 22604; + rev = 2; + hash = "sha256-ci8WE23fZ+ltEKFoUdNNVXsUIV0jqtas79ia2lYIo88="; + }; + + # Dashboard 15362 ("ZFS Pool Metrics") used to fill this slot. It is an *InfluxDB* + # dashboard -- its queries are InfluxQL against a `zpool_stats` measurement -- so it + # can never render against Prometheus, however it is provisioned. This is a + # Prometheus-native replacement built on the metrics we actually export: node + # exporter's zfs collector plus the ZFS textfile collector. + # Generated by dashboards/gen_zfs.py. + zfs-pools = fixup "zfs-pools" ../dashboards/zfs-pools.json; + + # CrystalDiskInfo-style per-drive SMART view. Generated by dashboards/gen_smart.py. + # The Good/Caution/Bad verdict and the per-drive row titles both come from the + # recording rules in rules/smartctl.yml, so this one needs those loaded to be useful. + drive-health = fixup "drive-health" ../dashboards/smart.json; +} diff --git a/modules/default.nix b/modules/default.nix new file mode 100644 index 0000000..07aa854 --- /dev/null +++ b/modules/default.nix @@ -0,0 +1,13 @@ +# Both halves at once, for running collectors and the web services on one machine. +# +# Importing this is equivalent to importing collectors.nix and server.nix together. If +# you are splitting them -- collectors on the host, server in a container -- import the +# two modules separately instead; each is self-contained. +{ ... }: + +{ + imports = [ + ./collectors.nix + ./server.nix + ]; +} diff --git a/modules/packages.nix b/modules/packages.nix new file mode 100644 index 0000000..5e76f10 --- /dev/null +++ b/modules/packages.nix @@ -0,0 +1,45 @@ +# The two textfile collectors, as ordinary packages. +# +# They are wrappers rather than inline `writeShellApplication` bodies in the module so +# that the Python is runnable by hand -- `zfs-metrics --output /tmp/x.prom` behaves +# exactly as the timer does, which is how every one of these metrics was validated. +# Neither wrapper hardcodes an output path; the unit passes it. +{ lib, writeShellApplication, python3, smartmontools, zfs }: + +{ + # Pool capacity and health, vdev errors, scrub history, dataset properties and + # snapshot retention -- none of which node_exporter's built-in zfs collector reports, + # because it reads /proc/spl/kstat/zfs and that only carries ARC statistics and objset + # counters. OpenZFS 2.2+ also dropped the per-pool `io` kstat, so node_zfs_zpool_* is + # largely empty. This shells out to the zpool/zfs CLIs instead. + # + # runtimeInputs is the whole reason for the wrapper: the collector must run against + # the same zfs userland as the running kernel module. + zfs-metrics = writeShellApplication { + name = "zfs-metrics"; + runtimeInputs = [ zfs ]; + text = '' + exec ${python3.interpreter} ${../collectors/zfs-metrics.py} "$@" + ''; + meta = { + description = "Export ZFS pool, dataset and snapshot metrics in Prometheus format"; + mainProgram = "zfs-metrics"; + }; + }; + + # smartctl_exporter reads only the entry *count* out of each drive's self-test log and + # discards the log table, so it cannot say when a drive last ran a short or extended + # test or whether it passed. It also has no way to render a drive's identity as a + # display string, which the dashboards need for row titles. Both come from here. + smart-selftest = writeShellApplication { + name = "smart-selftest"; + runtimeInputs = [ smartmontools ]; + text = '' + exec ${python3.interpreter} ${../collectors/smart-selftest.py} "$@" + ''; + meta = { + description = "Export SMART self-test history and drive identity in Prometheus format"; + mainProgram = "smart-selftest"; + }; + }; +} diff --git a/modules/server.nix b/modules/server.nix new file mode 100644 index 0000000..1996f5e --- /dev/null +++ b/modules/server.nix @@ -0,0 +1,420 @@ +# The web half: Prometheus, Loki, Grafana and an optional reverse proxy. +# +# Nothing here reads host hardware, so this is the part you can put in a container. It +# reaches the collectors over the network -- see `collectorHosts`. +{ config, lib, pkgs, ... }: + +let + inherit (lib) mkIf mkOption mkEnableOption mkDefault types optionals filterAttrs mapAttrs' nameValuePair; + + cfg = config.services.telemetry.server; + ports = config.services.telemetry.ports; + + # Fixed, because the generated dashboards reference them literally. dashboards/panels.py + # emits {"type": "prometheus", "uid": "prometheus"} on every panel, and the jq fixup in + # dashboards.nix rewrites grafana.com's placeholders to the same values. Changing these + # would silently break every panel, so they are not options. + dsUid = { prometheus = "prometheus"; loki = "loki"; }; + + # A `host` label carrying the machine name with the port stripped off. + # + # `instance` cannot do this job. The same machine is scraped twice on different ports + # -- the node exporter on :9100 serves the textfile metrics (zfs_*, smart_selftest_*, + # smart_drive_label) while smartctl_* comes from its own exporter on :9633 -- so one + # machine has two `instance` values, and the recording rules join straight across that + # boundary. `host` is equal on both, which is what makes the dashboards' host picker + # and those joins work at the same time. + # + # Note the regex is fully anchored by Prometheus, and only handles host:port. A + # bracketed IPv6 literal needs an explicit label via `extraScrapeConfigs` instead. + hostLabelRelabel = [{ + source_labels = [ "__address__" ]; + regex = "([^:]+)(?::[0-9]+)?"; + target_label = "host"; + replacement = "$1"; + }]; + + builtinDashboards = pkgs.callPackage ./dashboards.nix { inherit dsUid; }; + + selected = filterAttrs (name: _: cfg.grafana.dashboards.${name} or false) builtinDashboards; + + dashboards = selected // cfg.grafana.extraDashboards; + + dashboardOption = description: mkOption { + inherit description; + type = types.bool; + default = true; + }; +in +{ + imports = [ ./common.nix ]; + + options.services.telemetry.server = { + enable = mkEnableOption "the Prometheus/Loki/Grafana server"; + + collectorHosts = mkOption { + type = types.listOf types.str; + default = [ "localhost" ]; + example = [ "poweredge" "optiplex" ]; + description = '' + Machines running modules/collectors.nix. Each is scraped on both the node and + smartctl ports from {option}`services.telemetry.ports`, so this is normally the + only thing on the server side that has to know about the hosts. + + Each entry also becomes the value of the `host` label on everything scraped + from that machine, which is what the dashboards' host picker selects on. Use + the same spelling you would type in a browser; if you list an IP address, that + is what the picker will show. + + Set {option}`services.telemetry.server.prometheus.nodeTargets` or + {option}`services.telemetry.server.prometheus.smartctlTargets` to take over + target generation entirely. + ''; + }; + + listenAddress = mkOption { + type = types.str; + default = "0.0.0.0"; + description = '' + Address Loki, Prometheus and Grafana bind to. Loki has to be reachable from + every host running the collectors, so this defaults to all interfaces. + ''; + }; + + openFirewall = mkOption { + type = types.bool; + default = true; + description = '' + Open the ports of whichever services are enabled: Loki (the collectors push to + it), Prometheus (useful for checking scrape health directly), and either the + proxy or Grafana. + ''; + }; + + prometheus = { + enable = mkOption { + type = types.bool; + default = true; + description = "Run Prometheus."; + }; + + scrapeInterval = mkOption { + type = types.str; + default = "15s"; + description = '' + Global scrape interval, and the value handed to Grafana's Prometheus + datasource as `timeInterval`. + + These two MUST agree. Grafana derives `$__rate_interval` from `timeInterval`, + and if it guesses wrong then `rate()` windows are mis-sized and every + rate-based panel intermittently renders "No data" -- which is exactly what + Prometheus' own 1m default does to the Node Exporter Full dashboard. Setting + it here sets both. + ''; + }; + + nodeTargets = mkOption { + type = types.listOf types.str; + default = map (host: "${host}:${toString ports.node}") cfg.collectorHosts; + defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the node exporter port"; + description = "Targets for the `node` scrape job."; + }; + + smartctlTargets = mkOption { + type = types.listOf types.str; + default = map (host: "${host}:${toString ports.smartctl}") cfg.collectorHosts; + defaultText = lib.literalMD "one target per entry in `collectorHosts`, on the smartctl exporter port"; + description = "Targets for the `smartctl` scrape job. Empty disables the job."; + }; + + smartctlScrapeInterval = mkOption { + type = types.str; + default = "60s"; + description = '' + Scrape interval for the smartctl job, which opts out of the global one: + querying SMART is slow and the data barely moves. Safe because the drive + dashboards use no `rate()` queries at all. + ''; + }; + + ruleFiles = mkOption { + type = types.listOf types.path; + default = [ ../rules/smartctl.yml ]; + defaultText = lib.literalExpression "[ ../rules/smartctl.yml ]"; + description = '' + Recording rule files. The default computes the per-drive Good/Caution/Bad + verdict and the row-title labels that the drive health dashboard displays, so + drop it only if you also drop that dashboard. + + {option}`services.prometheus.checkConfig` runs promtool over these at build + time, so a broken rule fails the build rather than the service. + ''; + }; + + extraScrapeConfigs = mkOption { + type = types.listOf types.attrs; + default = [ ]; + description = "Additional scrape jobs, appended to the generated ones."; + }; + + retentionTime = mkOption { + type = types.str; + default = "90d"; + description = "How long to keep samples."; + }; + }; + + loki = { + enable = mkOption { + type = types.bool; + default = true; + description = "Run Loki to receive journal logs from the collectors."; + }; + + dataDir = mkOption { + type = types.path; + default = "/var/lib/loki"; + description = "Where Loki stores its chunks and index."; + }; + }; + + grafana = { + enable = mkOption { + type = types.bool; + default = true; + description = "Run Grafana, provisioned with datasources and dashboards."; + }; + + secretKeyFile = mkOption { + type = types.nullOr types.path; + default = null; + example = lib.literalExpression "config.sops.secrets.grafana-secret-key.path"; + description = '' + File holding Grafana's `security.secret_key`. Loaded through systemd's + `LoadCredential`, so the file only has to be readable by root at start time + and never lands in the Nix store. + + Deliberately a plain path rather than a sops-nix integration: wire it up to + whatever secret manager you already use. + ''; + }; + + dashboards = { + node-exporter-full = dashboardOption "Provision grafana.com dashboard 1860 (Node Exporter Full)."; + smartctl-exporter = dashboardOption "Provision grafana.com dashboard 22604 (Smartctl Exporter)."; + zfs-pools = dashboardOption "Provision the ZFS pools dashboard. Needs the ZFS collector."; + drive-health = dashboardOption "Provision the CrystalDiskInfo-style drive health dashboard. Needs the SMART collector and the smartctl recording rules."; + }; + + extraDashboards = mkOption { + type = types.attrsOf types.path; + default = { }; + example = lib.literalExpression ''{ my-dashboard = ./my-dashboard.json; }''; + description = '' + Extra dashboards to provision, as { name = path-to-json; }. These are passed + through unmodified, so they must already reference the datasource UIDs + "prometheus" and "loki". + ''; + }; + }; + + proxy = { + enable = mkOption { + type = types.bool; + default = false; + description = '' + Run Caddy in front of Grafana. Off by default because a reverse proxy is + usually something you already have; turn it on to get Grafana on port 80 + without exposing its own port. + ''; + }; + + virtualHost = mkOption { + type = types.str; + default = ":80"; + example = "metrics.example.com"; + description = '' + Caddy virtual host. A bare hostname makes Caddy provision a certificate + automatically; ":80" serves plain HTTP on every name. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + # Grafana uses this key to encrypt datasource credentials in its database, and + # nixpkgs stopped shipping a default in 26.05. Caught here rather than left to + # Grafana's own assertion, which names an option this module manages for you. + assertions = [{ + assertion = !cfg.grafana.enable || cfg.grafana.secretKeyFile != null; + message = '' + services.telemetry.server.grafana.secretKeyFile must be set when Grafana is + enabled. Generate a key (`openssl rand -base64 32`), store it with whatever + secret manager you use, and point this option at the resulting file. + ''; + }]; + + # -------------------------------------------------------------------------------- + # Loki + # -------------------------------------------------------------------------------- + services.loki = mkIf cfg.loki.enable { + enable = true; + configuration = { + auth_enabled = false; + + server = { + http_listen_port = ports.loki; + # Explicit: the collectors have to reach this from another host or container. + http_listen_address = cfg.listenAddress; + }; + + common = { + path_prefix = cfg.loki.dataDir; + storage.filesystem = { + chunks_directory = "${cfg.loki.dataDir}/chunks"; + rules_directory = "${cfg.loki.dataDir}/rules"; + }; + ring = { + instance_addr = "127.0.0.1"; + kvstore.store = "inmemory"; + }; + replication_factor = 1; + }; + + schema_config.configs = [{ + from = "2024-01-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { prefix = "index_"; period = "24h"; }; + }]; + }; + }; + + # -------------------------------------------------------------------------------- + # Prometheus + # -------------------------------------------------------------------------------- + services.prometheus = mkIf cfg.prometheus.enable { + enable = true; + port = ports.prometheus; + listenAddress = cfg.listenAddress; + retentionTime = cfg.prometheus.retentionTime; + + globalConfig = { + scrape_interval = cfg.prometheus.scrapeInterval; + evaluation_interval = cfg.prometheus.scrapeInterval; + scrape_timeout = "10s"; + }; + + inherit (cfg.prometheus) ruleFiles; + + scrapeConfigs = + optionals (cfg.prometheus.nodeTargets != [ ]) [{ + job_name = "node"; + relabel_configs = hostLabelRelabel; + static_configs = [{ targets = cfg.prometheus.nodeTargets; }]; + }] + ++ optionals (cfg.prometheus.smartctlTargets != [ ]) [{ + job_name = "smartctl"; + scrape_interval = cfg.prometheus.smartctlScrapeInterval; + scrape_timeout = "30s"; + relabel_configs = hostLabelRelabel; + static_configs = [{ targets = cfg.prometheus.smartctlTargets; }]; + }] + ++ cfg.prometheus.extraScrapeConfigs; + }; + + # -------------------------------------------------------------------------------- + # Grafana + # -------------------------------------------------------------------------------- + services.grafana = mkIf cfg.grafana.enable { + enable = true; + + settings = { + server = { + http_addr = mkDefault cfg.listenAddress; + http_port = mkDefault ports.grafana; + }; + } + # A file provider rather than a literal, so the key is never written into a + # world-readable store path. The credential is mounted by LoadCredential below. + // lib.optionalAttrs (cfg.grafana.secretKeyFile != null) { + security.secret_key = "$__file{/run/credentials/grafana.service/secret_key}"; + }; + + provision = { + enable = true; + + dashboards.settings.providers = [{ + name = "provisioned"; + disableDeletion = true; + options = { + path = "/etc/grafana-dashboards"; + foldersFromFilesStructure = true; + }; + }]; + + # Grafana cannot change the UID of an existing datasource in place. Its + # provisioner finds the datasource by name, then calls UpdateDataSource, which + # re-fetches using `id AND uid` -- with a new uid that row does not match, so it + # fails with "data source not found" and Grafana refuses to start. Deleting + # first turns the upsert into a plain insert; the provisioner runs deletes + # before inserts for exactly this case and suppresses the deletion event because + # the datasource is about to be re-created. Safe permanently: both datasources + # are fully declarative and dashboards bind to them by UID, not database ID. + datasources.settings.deleteDatasources = + [{ name = "Prometheus"; orgId = 1; }] + ++ optionals cfg.loki.enable [{ name = "Loki"; orgId = 1; }]; + + datasources.settings.datasources = + optionals cfg.prometheus.enable [{ + name = "Prometheus"; + type = "prometheus"; + uid = dsUid.prometheus; + url = "http://127.0.0.1:${toString ports.prometheus}"; + isDefault = true; + jsonData = { + # What $__rate_interval is derived from. Leaving it unset makes Grafana + # assume 15s and mis-size every rate() window. See scrapeInterval above. + timeInterval = cfg.prometheus.scrapeInterval; + httpMethod = "POST"; + }; + }] + ++ optionals cfg.loki.enable [{ + name = "Loki"; + type = "loki"; + uid = dsUid.loki; + url = "http://127.0.0.1:${toString ports.loki}"; + }]; + }; + }; + + systemd.services.grafana = mkIf (cfg.grafana.enable && cfg.grafana.secretKeyFile != null) { + serviceConfig.LoadCredential = [ "secret_key:${cfg.grafana.secretKeyFile}" ]; + }; + + environment.etc = mkIf cfg.grafana.enable ( + mapAttrs' (name: src: nameValuePair "grafana-dashboards/${name}.json" { source = src; }) + dashboards + ); + + # -------------------------------------------------------------------------------- + # Reverse proxy + # -------------------------------------------------------------------------------- + services.caddy = mkIf cfg.proxy.enable { + enable = true; + virtualHosts.${cfg.proxy.virtualHost}.extraConfig = '' + reverse_proxy localhost:${toString ports.grafana} + ''; + }; + + # -------------------------------------------------------------------------------- + # Firewall + # -------------------------------------------------------------------------------- + networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall ( + optionals cfg.loki.enable [ ports.loki ] + ++ optionals cfg.prometheus.enable [ ports.prometheus ] + ++ (if cfg.proxy.enable then [ 80 443 ] else optionals cfg.grafana.enable [ ports.grafana ]) + ); + }; +} diff --git a/rules/smartctl.yml b/rules/smartctl.yml index dafed8d..3982e06 100644 --- a/rules/smartctl.yml +++ b/rules/smartctl.yml @@ -5,7 +5,7 @@ # # The fleet is mixed: SATA drives expose the classic ATA attribute table, SAS drives # expose none of it and report health through SCSI error counters instead. Every rule -# below is therefore made *total* -- it ends in `or (count by (device) (...) * 0)` so +# below is therefore made *total* -- it ends in `or (count by (host, device) (...) * 0)` so # that each series exists with value 0 for every drive, including drives of the other # kind. Without that, the arithmetic in smartctl:health_code would silently drop drives # whose label sets did not match on both sides. @@ -20,12 +20,12 @@ groups: # fails", so those are excluded rather than treated as an instant failure. - record: smartctl:attr_failing:count expr: | - count by (device) ( + count by (host, device) ( smartctl_device_attribute{attribute_value_type="value"} - <= on(device, attribute_id) + <= on(host, device, attribute_id) (smartctl_device_attribute{attribute_value_type="thresh"} > 0) ) - or (count by (device) (smartctl_device_smart_status) * 0) + or (count by (host, device) (smartctl_device_smart_status) * 0) # Attributes whose raw value is a genuine warning sign across vendors: # 5 Reallocated_Sector_Ct 187 Reported_Uncorrect @@ -35,13 +35,13 @@ groups: # values on Seagate drives by design and would flag healthy disks. - record: smartctl:attr_caution:count expr: | - count by (device) ( + count by (host, device) ( smartctl_device_attribute{ attribute_value_type="raw", attribute_id=~"5|187|188|197|198|199" } > 0 ) - or (count by (device) (smartctl_device_smart_status) * 0) + or (count by (host, device) (smartctl_device_smart_status) * 0) # --- SAS / SCSI ----------------------------------------------------------- # Grown defect list: sectors reallocated since the drive left the factory. The @@ -49,15 +49,15 @@ groups: # acting on. - record: smartctl:scsi_defects:count expr: | - sum by (device) (smartctl_scsi_grown_defect_list) - or (count by (device) (smartctl_device_smart_status) * 0) + sum by (host, device) (smartctl_scsi_grown_defect_list) + or (count by (host, device) (smartctl_device_smart_status) * 0) # Errors the drive could not correct at all, which means data was lost. - record: smartctl:scsi_uncorrected:count expr: | - sum by (device) (smartctl_read_total_uncorrected_errors) - + sum by (device) (smartctl_write_total_uncorrected_errors) - or (count by (device) (smartctl_device_smart_status) * 0) + sum by (host, device) (smartctl_read_total_uncorrected_errors) + + sum by (host, device) (smartctl_write_total_uncorrected_errors) + or (count by (host, device) (smartctl_device_smart_status) * 0) # --- Verdict -------------------------------------------------------------- # 0 good, 1 caution, 2 bad. The inner clamp_max on each group keeps two @@ -66,7 +66,7 @@ groups: expr: | clamp_max( 2 * clamp_max( - (max by (device) (smartctl_device_smart_status) == bool 0) + (max by (host, device) (smartctl_device_smart_status) == bool 0) + (smartctl:attr_failing:count > bool 0) + (smartctl:scsi_uncorrected:count > bool 0) , 1) @@ -82,9 +82,9 @@ groups: # table, and only SCSI drives report a grown defect list. - record: smartctl:bus expr: | - label_replace(count by (device) (smartctl_device_attribute), "bus", "ata", "", "") + label_replace(count by (host, device) (smartctl_device_attribute), "bus", "ata", "", "") or - label_replace(count by (device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "") + label_replace(count by (host, device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "") # The identity string from collectors/smart-selftest.py, made total so a drive # still gets a row if that collector has not written its file yet. `unless` (not @@ -92,11 +92,11 @@ groups: # `or` would emit two series per drive rather than deduplicating. - record: smartctl:ident expr: | - max by (device, label) (smart_drive_label) + max by (host, device, label) (smart_drive_label) or label_replace( - count by (device) (smartctl_device_smart_status) - unless on(device) smart_drive_label, + count by (host, device) (smartctl_device_smart_status) + unless on(host, device) smart_drive_label, "label", "identity pending", "", "") # One preformatted line per drive: "sde · GOOD · 6.0 TB · SAS 7.2K · model · SN". @@ -111,6 +111,6 @@ groups: or label_replace(smartctl:health_code == 1, "verdict", "CAUTION", "", "") or label_replace(smartctl:health_code == 2, "verdict", "BAD", "", "") ) - * on(device) group_left(bus) smartctl:bus - * on(device) group_left(label) smartctl:ident, + * on(host, device) group_left(bus) smartctl:bus + * on(host, device) group_left(label) smartctl:ident, "row", " · ", "device", "verdict", "label") |