diff options
| author | Tim Keller <tjk@tjkeller.xyz> | 2026-09-02 21:01:46 -0500 |
|---|---|---|
| committer | Tim Keller <tjk@tjkeller.xyz> | 2026-09-02 21:01:46 -0500 |
| commit | e88a880435ebdda6e50bf16a493abb85b4e80d15 (patch) | |
| tree | cc59b2810868bcc90bb2772380945fc024f7f5ee /collectors | |
| parent | a74c2877571afbfd04584b8821c754e9050b9e72 (diff) | |
| download | telemetry-flake-flake-modules.tar.xz telemetry-flake-flake-modules.zip | |
add NVMe support, and give the self-test collector the device listHEADmasterflake-modules
Two configurations the modules claimed to support and did not.
NVMe drives had no per-drive section. smartctl:bus routes a drive by metric
presence -- ATA attribute table, or SCSI grown defect list -- and NVMe emits
neither, confirmed against a live smartctl_exporter 0.14.0 on real hardware.
With no bus label the smartctl:row join dropped them, so they appeared in Fleet
Overview and All Drives but could not be opened. Adds an nvme arm keyed on
smartctl_device_percentage_used, verdict rules over the SMART/Health
Information log, and a third repeating section.
The verdict is BAD on a critical warning, media errors, or available spare
below the drive's own threshold, and CAUTION at 90% of rated endurance.
num_err_log_entries is deliberately not in it: the NVMe drive in this machine
is healthy and sits at 2200, because it counts every command that ever
returned an error.
smart.devices reached the exporter but not the self-test collector, which
always ran its own `smartctl --scan` -- exactly what cannot see drives behind a
RAID controller, which is the only reason to set the option. The collector now
takes the same PATH[;TYPE] syntax and passes -d through.
That means it also has to reproduce the exporter's device label, which is not
the basename: /dev/bus/0 as megaraid,1 is bus_0_megaraid_1. Ported
buildDeviceLabel with its /dev/disk/by-{id,path}/ prefix stripping and the
comma rule, and the scan-time scsi -> auto fallback. Getting this wrong fails
silently -- every join in rules/smartctl.yml simply misses.
Verified: promtool; a Prometheus fixture covering all three transports plus a
megaraid-labelled device, asserting bus and verdict per drive and resolving
$ata_device/$sas_device/$nvme_device the way Grafana does; the grid layout;
the ExecStart with and without smart.devices; and the collector against this
machine's NVMe drive.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JKczN1f66kmLxNBLX37voQ
Diffstat (limited to 'collectors')
| -rw-r--r-- | collectors/smart-selftest.py | 78 |
1 files changed, 64 insertions, 14 deletions
diff --git a/collectors/smart-selftest.py b/collectors/smart-selftest.py index 6cb026c..dca7752 100644 --- a/collectors/smart-selftest.py +++ b/collectors/smart-selftest.py @@ -12,11 +12,13 @@ Two things smartctl_exporter cannot give us: title, and PromQL has no way to turn a number (capacity, rotation rate) into one. So the formatting happens here, in `smart_drive_label`. -Device names are emitted without the /dev/ prefix so they join against -smartctl_exporter's `device` label, which is the bare name ("sda"). +Everything here is labelled with the same `device` string smartctl_exporter uses, so the +two join. That is not simply the basename -- see device_label() below. -Run with --output PATH. Reading the self-test log does not spin an idle disk up, but -there is no reason to poll it often -- a 15 minute timer is plenty. +Run with --output PATH. Devices are discovered with `smartctl --scan` unless --device is +given; pass it in the exporter's own `PATH[;TYPE]` form, repeatably, so one list can feed +both. Reading the self-test log does not spin an idle disk up, but there is no reason to +poll it often -- a 15 minute timer is plenty. """ from __future__ import annotations @@ -108,11 +110,52 @@ def run_json(args: list[str]) -> dict | None: return None -def scan_devices() -> list[str]: +# Everything smartctl_exporter strips off the front of a device name before using it as +# a label. Kept identical to the exporter's own `devReg` (pkg/collector: buildDeviceLabel) +# because a mismatch here does not fail anything -- it just makes every join in +# rules/smartctl.yml miss, so drives lose their identity and self-test history silently. +DEV_PREFIX = re.compile(r"^/dev/(?:disk/by-id/|disk/by-path/|)") + + +def device_label(name: str, dtype: str) -> str: + """The `device` label smartctl_exporter would give this device. + + /dev/sda -> sda + /dev/nvme0 -> nvme0 + /dev/bus/0 as megaraid,1 -> bus_0_megaraid_1 + + The type is only appended when it contains a comma, which is what distinguishes a + controller-addressed device (megaraid,1 / cciss,0) -- where the path alone is + ambiguous because several drives share it -- from a plain one (sat, nvme, auto). + """ + base = DEV_PREFIX.sub("", name).replace("/", "_") + if "," in dtype: + return f"{base}_{dtype.replace(',', '_')}" + return base + + +def parse_device_arg(spec: str) -> tuple[str, str]: + """Split the exporter's `PATH[;TYPE]` device syntax, e.g. '/dev/bus/0;megaraid,1'.""" + name, _, dtype = spec.partition(";") + return name, dtype or "auto" + + +def scan_devices() -> list[tuple[str, str]]: data = run_json(["smartctl", "--scan", "-j"]) if not data: return [] - return [d["name"] for d in data.get("devices", []) if d.get("name")] + devices = [] + for d in data.get("devices", []): + if not d.get("name"): + continue + dtype = d.get("type", "") + # --scan reports SATA disks as "scsi"; taking that literally makes smartctl talk + # plain SCSI to them and return no ATA attribute table. The exporter falls back + # to auto-detection here, so do the same or the two disagree about the drive. + if dtype == "scsi": + dtype = "auto" + devices.append((d["name"], dtype)) + return devices def normalise_type(text: str) -> str: @@ -254,14 +297,19 @@ def collect_identity(reg: Registry, name: str, data: dict) -> None: "Drive identity rendered for display; the value is always 1.") -def collect_device(reg: Registry, dev: str) -> bool: +def collect_device(reg: Registry, dev: str, dtype: str) -> bool: # -i rides along on the same invocation, so identity costs no extra smartctl run # and no extra chance of waking an idle disk. - data = run_json(["smartctl", "-j", "-i", "-l", "selftest", dev]) + args = ["smartctl", "-j", "-i", "-l", "selftest"] + # -d auto is smartctl's own default; passing it explicitly only adds a way for an + # old smartctl to reject the invocation. + if dtype and dtype != "auto": + args += ["-d", dtype] + data = run_json(args + [dev]) if data is None: return False - name = os.path.basename(dev) + name = device_label(dev, dtype) collect_identity(reg, name, data) counts: dict[tuple[str, str], int] = defaultdict(int) totals: dict[str, int] = defaultdict(int) @@ -307,21 +355,23 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", required=True, help="path of the .prom file to write") - parser.add_argument("--device", action="append", default=[], - help="device to inspect; repeatable. Default: smartctl --scan") + parser.add_argument("--device", action="append", default=[], metavar="PATH[;TYPE]", + help="device to inspect, in smartctl_exporter's syntax, e.g. " + "'/dev/bus/0;megaraid,1'; repeatable. " + "Default: smartctl --scan") args = parser.parse_args() reg = Registry() started = time.time() - devices = args.device or scan_devices() + devices = [parse_device_arg(d) for d in args.device] or scan_devices() if not devices: warn("no devices found") seen = 0 - for dev in devices: + for dev, dtype in devices: try: - if collect_device(reg, dev): + if collect_device(reg, dev, dtype): seen += 1 except Exception as exc: # noqa: BLE001 - one bad disk must not lose the rest warn(f"{dev}: {exc!r}") |
