diff options
| -rw-r--r-- | README.md | 10 | ||||
| -rw-r--r-- | collectors/smart-selftest.py | 78 | ||||
| -rw-r--r-- | dashboards/gen_smart.py | 154 | ||||
| -rw-r--r-- | modules/collectors.nix | 23 | ||||
| -rw-r--r-- | rules/smartctl.yml | 40 |
5 files changed, 259 insertions, 46 deletions
@@ -18,8 +18,8 @@ What *has* been checked: - every dashboard query executed against a real Prometheus fed fixture data - `promtool` over the recording rules and the generated scrape config - `nix flake check`: both modules evaluate, and both dashboard generators run -- the collectors run against mock `smartctl`/`zpool`/`zfs`, and partially against real - hardware +- the collectors run against mock `smartctl`/`zpool`/`zfs`, and against a real NVMe drive +- the NVMe metric names the rules depend on, checked against a live `smartctl_exporter` What has **not**: @@ -156,7 +156,7 @@ host and the whole page swaps to it. | dashboard | picker(s) | |---|---| -| Drive Health | **Host** → SATA drive / SAS drive | +| Drive Health | **Host** → SATA drive / SAS drive / NVMe drive | | ZFS Pools | **Host** → Pool | | Node Exporter Full | Job → Nodename → Instance | | Smartctl Exporter | Node | @@ -233,8 +233,8 @@ Shipped in `rules/alerts.yml`, loaded unless you set `alerts.defaultRules = fals | | condition | severity | |---|---|---| -| `DriveFailing` | SMART self-assessment failed, an attribute is below its threshold, or uncorrectable errors | critical | -| `DriveCaution` | reallocated/pending sectors, or a non-empty SAS grown defect list | warning | +| `DriveFailing` | SMART self-assessment failed, an attribute is below its threshold, uncorrectable errors, or an NVMe critical warning | critical | +| `DriveCaution` | reallocated/pending sectors, a non-empty SAS grown defect list, or 90%+ NVMe endurance used | warning | | `SelfTestFailed` | last short or extended self-test did not pass | critical | | `DriveOverheating` | above 55 °C for 30 minutes | warning | | `PoolNotOnline` | pool is DEGRADED, FAULTED, … | critical | diff --git a/collectors/smart-selftest.py b/collectors/smart-selftest.py index 6cb026c..dca7752 100644 --- a/collectors/smart-selftest.py +++ b/collectors/smart-selftest.py @@ -12,11 +12,13 @@ Two things smartctl_exporter cannot give us: title, and PromQL has no way to turn a number (capacity, rotation rate) into one. So the formatting happens here, in `smart_drive_label`. -Device names are emitted without the /dev/ prefix so they join against -smartctl_exporter's `device` label, which is the bare name ("sda"). +Everything here is labelled with the same `device` string smartctl_exporter uses, so the +two join. That is not simply the basename -- see device_label() below. -Run with --output PATH. Reading the self-test log does not spin an idle disk up, but -there is no reason to poll it often -- a 15 minute timer is plenty. +Run with --output PATH. Devices are discovered with `smartctl --scan` unless --device is +given; pass it in the exporter's own `PATH[;TYPE]` form, repeatably, so one list can feed +both. Reading the self-test log does not spin an idle disk up, but there is no reason to +poll it often -- a 15 minute timer is plenty. """ from __future__ import annotations @@ -108,11 +110,52 @@ def run_json(args: list[str]) -> dict | None: return None -def scan_devices() -> list[str]: +# Everything smartctl_exporter strips off the front of a device name before using it as +# a label. Kept identical to the exporter's own `devReg` (pkg/collector: buildDeviceLabel) +# because a mismatch here does not fail anything -- it just makes every join in +# rules/smartctl.yml miss, so drives lose their identity and self-test history silently. +DEV_PREFIX = re.compile(r"^/dev/(?:disk/by-id/|disk/by-path/|)") + + +def device_label(name: str, dtype: str) -> str: + """The `device` label smartctl_exporter would give this device. + + /dev/sda -> sda + /dev/nvme0 -> nvme0 + /dev/bus/0 as megaraid,1 -> bus_0_megaraid_1 + + The type is only appended when it contains a comma, which is what distinguishes a + controller-addressed device (megaraid,1 / cciss,0) -- where the path alone is + ambiguous because several drives share it -- from a plain one (sat, nvme, auto). + """ + base = DEV_PREFIX.sub("", name).replace("/", "_") + if "," in dtype: + return f"{base}_{dtype.replace(',', '_')}" + return base + + +def parse_device_arg(spec: str) -> tuple[str, str]: + """Split the exporter's `PATH[;TYPE]` device syntax, e.g. '/dev/bus/0;megaraid,1'.""" + name, _, dtype = spec.partition(";") + return name, dtype or "auto" + + +def scan_devices() -> list[tuple[str, str]]: data = run_json(["smartctl", "--scan", "-j"]) if not data: return [] - return [d["name"] for d in data.get("devices", []) if d.get("name")] + devices = [] + for d in data.get("devices", []): + if not d.get("name"): + continue + dtype = d.get("type", "") + # --scan reports SATA disks as "scsi"; taking that literally makes smartctl talk + # plain SCSI to them and return no ATA attribute table. The exporter falls back + # to auto-detection here, so do the same or the two disagree about the drive. + if dtype == "scsi": + dtype = "auto" + devices.append((d["name"], dtype)) + return devices def normalise_type(text: str) -> str: @@ -254,14 +297,19 @@ def collect_identity(reg: Registry, name: str, data: dict) -> None: "Drive identity rendered for display; the value is always 1.") -def collect_device(reg: Registry, dev: str) -> bool: +def collect_device(reg: Registry, dev: str, dtype: str) -> bool: # -i rides along on the same invocation, so identity costs no extra smartctl run # and no extra chance of waking an idle disk. - data = run_json(["smartctl", "-j", "-i", "-l", "selftest", dev]) + args = ["smartctl", "-j", "-i", "-l", "selftest"] + # -d auto is smartctl's own default; passing it explicitly only adds a way for an + # old smartctl to reject the invocation. + if dtype and dtype != "auto": + args += ["-d", dtype] + data = run_json(args + [dev]) if data is None: return False - name = os.path.basename(dev) + name = device_label(dev, dtype) collect_identity(reg, name, data) counts: dict[tuple[str, str], int] = defaultdict(int) totals: dict[str, int] = defaultdict(int) @@ -307,21 +355,23 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", required=True, help="path of the .prom file to write") - parser.add_argument("--device", action="append", default=[], - help="device to inspect; repeatable. Default: smartctl --scan") + parser.add_argument("--device", action="append", default=[], metavar="PATH[;TYPE]", + help="device to inspect, in smartctl_exporter's syntax, e.g. " + "'/dev/bus/0;megaraid,1'; repeatable. " + "Default: smartctl --scan") args = parser.parse_args() reg = Registry() started = time.time() - devices = args.device or scan_devices() + devices = [parse_device_arg(d) for d in args.device] or scan_devices() if not devices: warn("no devices found") seen = 0 - for dev in devices: + for dev, dtype in devices: try: - if collect_device(reg, dev): + if collect_device(reg, dev, dtype): seen += 1 except Exception as exc: # noqa: BLE001 - one bad disk must not lose the rest warn(f"{dev}: {exc!r}") diff --git a/dashboards/gen_smart.py b/dashboards/gen_smart.py index 76ca8fe..d7b0671 100644 --- a/dashboards/gen_smart.py +++ b/dashboards/gen_smart.py @@ -9,15 +9,20 @@ it produces: A CrystalDiskInfo-style view: a Good/Caution/Bad verdict per drive, the full raw SMART attribute table, and self-test history. -The fleet is mixed, so there are two repeating sections: +The fleet is mixed, so there are three repeating sections, because the three transports +report health in three unrelated ways: * SATA drives expose the classic ATA attribute table (value / worst / thresh / raw). * SAS drives expose none of it. Their health lives in SCSI error counters instead -- grown defect list, uncorrected errors, corrected-error breakdown. +* NVMe drives expose neither, only the SMART/Health Information log -- a critical + warning bitfield, media errors, endurance used and the spare block pool. Drives are routed into the right section by *metric presence* rather than by the `protocol` label, which is more robust: only ATA drives have smartctl_device_attribute, -and only SCSI drives have smartctl_scsi_grown_defect_list. +only SCSI drives have smartctl_scsi_grown_defect_list, and the exporter emits +smartctl_device_percentage_used only for nvme. A drive matching none of the three gets +no section at all, so adding a transport means adding an arm to smartctl:bus. Metrics used: @@ -63,6 +68,7 @@ PASS_MAP = [{ HOST = 'host=~"$host"' ATA = f'{HOST}, device=~"$ata_device"' SAS = f'{HOST}, device=~"$sas_device"' +NVME = f'{HOST}, device=~"$nvme_device"' # The per-drive sections come from smartctl:row, one preformatted line per drive. # @@ -149,16 +155,17 @@ panels.append(stat( "Healthy", [(f'count(smartctl:health_code{{{HOST}}} == 0) or vector(0)', "drives")], gp(0, 1, 4, 4), colormode="background", thresholds={"mode": "absolute", "steps": [{"color": "green", "value": None}]}, - desc="Drives with no failing attribute, no SCSI uncorrected errors and a passing " - "SMART self-assessment.", + desc="Drives with no failing attribute, no SCSI uncorrected errors, no NVMe " + "critical warning and a passing SMART self-assessment.", )) panels.append(stat( "Caution", [(f'count(smartctl:health_code{{{HOST}}} == 1) or vector(0)', "drives")], gp(4, 1, 4, 4), colormode="background", thresholds={"mode": "absolute", "steps": [ {"color": "green", "value": None}, {"color": "orange", "value": 1}]}, - desc="Reallocated / pending / uncorrectable sectors on SATA, or a non-empty grown " - "defect list on SAS. Not failing yet — worth watching the trend.", + desc="Reallocated / pending / uncorrectable sectors on SATA, a non-empty grown " + "defect list on SAS, or 90%+ of rated endurance used on NVMe. Not failing yet " + "— worth watching the trend.", )) panels.append(stat( "Failing", [(f'count(smartctl:health_code{{{HOST}}} == 2) or vector(0)', "drives")], @@ -436,30 +443,133 @@ panels.append(timeseries( )) # ====================================================================================== +# NVMe — repeats once per NVMe drive +# ====================================================================================== +panels.append(row("${nvme_device:text}", 53, repeat="nvme_device")) + +panels.append(stat( + "Health", [(f'smartctl:health_code{{{NVME}}}', "{{device}}")], + gp(0, 54, 6, 4), mappings=HEALTH_MAP, thresholds=HEALTH_THRESHOLDS, + colormode="background", +)) +panels.append(stat( + "Temperature", + [(f'smartctl_device_temperature{{{NVME}, temperature_type="current"}}', "temp")], + gp(6, 54, 3, 4), unit="celsius", + thresholds={"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 50}, + {"color": "red", "value": 70}, + ]}, + desc="NVMe drives run hotter than spinning disks by design and throttle rather " + "than fail, so the thresholds here are higher than on the other sections.", +)) +panels.append(stat( + "Powered On", [(power_on(NVME), "hours")], + gp(9, 54, 3, 4), unit="s", +)) +panels.append(stat( + # smartctl reports user_capacity only for a single-namespace drive; with several + # namespaces the exporter publishes nvme_total_capacity instead and leaves this at + # zero, so filter the zero out rather than displaying it. + "Capacity", + [(f'max by (device) (smartctl_device_capacity_bytes{{{NVME}}} > 0 ' + f'or smartctl_device_nvme_capacity_bytes{{{NVME}}})', "bytes")], + gp(12, 54, 3, 4), unit="bytes", +)) +panels.append(stat( + "Wear", [(f'smartctl_device_percentage_used{{{NVME}}}', "used")], + gp(15, 54, 3, 4), unit="percent", colormode="background", + thresholds={"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 80}, + {"color": "red", "value": 100}, + ]}, + desc="The controller's estimate of consumed write endurance. 100% means the drive " + "has reached its rated lifetime, not that it is about to fail — plenty run " + "well past it. Watch how fast it climbs rather than the number.", +)) +panels.append(stat( + "Available Spare", [(f'smartctl_device_available_spare{{{NVME}}}', "remaining")], + gp(18, 54, 3, 4), unit="percent", colormode="background", + thresholds={"mode": "absolute", "steps": [ + {"color": "red", "value": None}, + {"color": "orange", "value": 15}, + {"color": "green", "value": 25}, + ]}, + desc="Spare blocks left for remapping, as a percentage of the factory pool. Falling " + "below the drive's own threshold — typically 5–10%, and reported separately as " + "smartctl_device_available_spare_threshold — sets a critical warning and is " + "treated as a failure here.", +)) +panels.append(stat( + "Media Errors", [(f'smartctl_device_media_errors{{{NVME}}}', "errors")], + gp(21, 54, 3, 4), thresholds=OK_BAD, colormode="background", + desc="Unrecovered data integrity errors — the NVMe equivalent of SAS uncorrected " + "errors. Should be zero.", +)) + +panels.append(drive_info( + "Drive — $nvme_device", NVME, gp(0, 58, 24, 4), + label_cols=["model_name", "serial_number", "firmware_version", "interface", + "protocol", "form_factor"], + renames={"model_name": "Model", "serial_number": "Serial", + "firmware_version": "Firmware", "interface": "Interface", + "protocol": "Bus", "form_factor": "Form"}, + desc="Identity of this drive. NVMe reports no rotation rate, so RPM always reads " + "SSD, and no model family or ATA/SATA version.", +)) + +panels.append(timeseries( + "Wear & Spare — $nvme_device", [ + (f'smartctl_device_percentage_used{{{NVME}}}', "endurance used"), + (f'smartctl_device_available_spare{{{NVME}}}', "spare remaining"), + (f'smartctl_device_available_spare_threshold{{{NVME}}}', "spare threshold"), + ], + gp(0, 62, 12, 8), unit="percent", minval=0, + custom={"lineWidth": 2, "fillOpacity": 0}, + desc="The two lines that matter are spare remaining and its threshold: the drive " + "declares itself failing when they cross.", +)) +panels.append(timeseries( + "Errors — $nvme_device", [ + (f'smartctl_device_critical_warning{{{NVME}}}', "critical warning"), + (f'smartctl_device_media_errors{{{NVME}}}', "media errors"), + (f'smartctl_device_num_err_log_entries{{{NVME}}}', "error log entries"), + ], + gp(12, 62, 12, 8), minval=0, + custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, + desc="Critical warning is a bitfield — any non-zero value is the drive asking to be " + "replaced. Error log entries count every command that ever returned an error, " + "including entirely harmless ones: a healthy consumer SSD can sit at a few " + "thousand. It is context, not a fault, which is why the verdict ignores it.", +)) + +# ====================================================================================== # Self-tests # ====================================================================================== -panels.append(row("Self-Tests", 53)) +panels.append(row("Self-Tests", 70)) panels.append(stat( "Drives Never Tested", [(f'count(smartctl_device_smart_status{{{HOST}}}) - ' f'count(count by (device) (smart_selftest_last_lifetime_hours{{{HOST}}})) or ' f'count(smartctl_device_smart_status{{{HOST}}})', "drives")], - gp(0, 54, 4, 5), thresholds=OK_BAD, colormode="background", + gp(0, 71, 4, 5), thresholds=OK_BAD, colormode="background", desc="Drives with no self-test log entries at all. smartd only runs tests if its " "config includes an -s schedule; merely enabling the service does not.", )) panels.append(stat( "Failed Self-Tests", [(f'count(smart_selftest_last_passed{{{HOST}}} == 0) or vector(0)', "tests")], - gp(4, 54, 4, 5), thresholds=OK_BAD, colormode="background", + gp(4, 71, 4, 5), thresholds=OK_BAD, colormode="background", )) panels.append(stat( "Oldest Long Test", [(f'max({power_on()} / 3600 - on(device) group_right() ' f'smart_selftest_last_lifetime_hours{{{HOST}, test_type="extended"}})', "hours ago")], - gp(8, 54, 4, 5), unit="h", + gp(8, 71, 4, 5), unit="h", thresholds={"mode": "absolute", "steps": [ {"color": "green", "value": None}, {"color": "yellow", "value": 24 * 14}, @@ -495,7 +605,7 @@ panels.append(table( (_keyed(power_on() + ' / 3600 - on(device) group_right() ' f'smart_selftest_last_lifetime_hours{{{HOST}}}'), None), ], - gp(12, 54, 12, 10), keep="drive_test|device|test_type|Value.*", + gp(12, 71, 12, 10), keep="drive_test|device|test_type|Value.*", renames={ "__join__": "drive_test", "device": "Device", "test_type": "Test", "Value #A": "Last Result", "Value #B": "At Hours", @@ -523,26 +633,26 @@ panels.append(table( # ====================================================================================== # Collapsed: trends and diagnostics # ====================================================================================== -panels.append(row("Attribute Trends", 64, collapsed=True, children=[ +panels.append(row("Attribute Trends", 81, collapsed=True, children=[ timeseries( "Reallocated Sectors (all SATA drives)", [(f'smartctl_device_attribute{{{HOST}, attribute_id="5", ' f'attribute_value_type="raw"}}', "{{device}}")], - gp(0, 65, 8, 7), minval=0, + gp(0, 82, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, ), timeseries( "Pending & Uncorrectable Sectors", [(f'smartctl_device_attribute{{{HOST}, attribute_id=~"197|198", ' f'attribute_value_type="raw"}}', "{{device}} {{attribute_name}}")], - gp(8, 65, 8, 7), minval=0, + gp(8, 82, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, ), timeseries( "CRC Errors (cabling)", [(f'smartctl_device_attribute{{{HOST}, attribute_id="199", ' f'attribute_value_type="raw"}}', "{{device}}")], - gp(16, 65, 8, 7), minval=0, + gp(16, 82, 8, 7), minval=0, custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, desc="Attribute 199 counts errors on the SATA link rather than the platters. " "Growth here usually means a cable or backplane, not a dying disk.", @@ -552,21 +662,21 @@ panels.append(row("Attribute Trends", 64, collapsed=True, children=[ (f'smartctl_device_bytes_written{{{HOST}}}', "{{device}} written"), (f'smartctl_device_bytes_read{{{HOST}}}', "{{device}} read"), ], - gp(0, 72, 12, 7), unit="bytes", minval=0, + gp(0, 89, 12, 7), unit="bytes", minval=0, ), timeseries( "Health Code History", [(f'smartctl:health_code{{{HOST}}}', "{{device}}")], - gp(12, 72, 12, 7), mappings=HEALTH_MAP, minval=0, maxval=2, + gp(12, 89, 12, 7), mappings=HEALTH_MAP, minval=0, maxval=2, custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"}, ), ])) -panels.append(row("Collector Health", 65, collapsed=True, children=[ +panels.append(row("Collector Health", 82, collapsed=True, children=[ timeseries( "smartctl Exit Status", [(f'smartctl_device_smartctl_exit_status{{{HOST}}}', "{{device}}")], - gp(0, 66, 8, 7), minval=0, + gp(0, 83, 8, 7), minval=0, desc="Non-zero means smartctl itself had trouble reading the device. Bit 0 is " "a command-line problem, bit 1 a device-open failure, bit 3 a failing " "drive.", @@ -574,14 +684,14 @@ panels.append(row("Collector Health", 65, collapsed=True, children=[ timeseries( "Self-Test Collector Duration", [(f'smart_selftest_collection_duration_seconds{{{HOST}}}', "collection")], - gp(8, 66, 8, 7), unit="s", minval=0, + gp(8, 83, 8, 7), unit="s", minval=0, ), timeseries( "Devices Seen", [ (f'smartctl_devices{{{HOST}}}', "exporter"), (f'smart_selftest_devices{{{HOST}}}', "self-test collector"), ], - gp(16, 66, 8, 7), minval=0, + gp(16, 83, 8, 7), minval=0, desc="These should agree. A divergence means one of the two cannot see some " "drives.", ), @@ -610,6 +720,8 @@ print(json.dumps(dashboard( label="SATA drive", regex=ROW_REGEX), query_var("sas_device", DEVICE_QUERY.format(bus="sas"), label="SAS drive", regex=ROW_REGEX), + query_var("nvme_device", DEVICE_QUERY.format(bus="nvme"), + label="NVMe drive", regex=ROW_REGEX), ], tags=["smart", "storage", "hardware"], ), indent=2)) diff --git a/modules/collectors.nix b/modules/collectors.nix index 6f328ee..879516b 100644 --- a/modules/collectors.nix +++ b/modules/collectors.nix @@ -148,11 +148,16 @@ in devices = mkOption { type = types.listOf types.str; default = [ ]; - example = [ "/dev/sda" "/dev/nvme0" ]; + example = [ "/dev/sda" "/dev/nvme0" "/dev/bus/0;megaraid,1" ]; description = '' - Devices to poll. Empty means auto-discovery via `smartctl --scan`, which is - correct for direct-attached AHCI/HBA disks. Behind a RAID controller you - generally have to list devices explicitly with their `-d` type. + Devices to poll, shared by the smartctl exporter and the self-test collector. + Empty means auto-discovery via `smartctl --scan`, which is correct for + direct-attached AHCI/HBA disks. Behind a RAID controller `--scan` returns + nothing useful and you have to list the drives explicitly, giving each one its + `-d` type after a semicolon: `"/dev/bus/0;megaraid,1"`. + + Note that setting this also turns off the exporter's periodic rescan, so + hot-swapped drives will not be picked up until it restarts. ''; }; @@ -268,7 +273,15 @@ in systemd.services.smart-selftest-collector = mkIf cfg.smart.enable (mkCollectorService { description = "Collect SMART self-test history and drive identity"; - execStart = "${lib.getExe collectorPkgs.smart-selftest} --output ${cfg.textfileDir}/smart-selftest.prom"; + # The same device list the exporter gets. Without this the collector would fall + # back to its own `smartctl --scan`, which is exactly what cannot see the drives + # in the one configuration the option exists for -- so every row title would read + # "identity pending" and self-test history would be empty. + execStart = lib.escapeShellArgs ([ + (lib.getExe collectorPkgs.smart-selftest) + "--output" + "${cfg.textfileDir}/smart-selftest.prom" + ] ++ concatMap (device: [ "--device" device ]) cfg.smart.devices); # Same shape as the smartctl exporter's own hardening: raw device access to talk # to the disks, and nothing else. extraServiceConfig = { diff --git a/rules/smartctl.yml b/rules/smartctl.yml index 3982e06..60a2caa 100644 --- a/rules/smartctl.yml +++ b/rules/smartctl.yml @@ -59,6 +59,36 @@ groups: + sum by (host, device) (smartctl_write_total_uncorrected_errors) or (count by (host, device) (smartctl_device_smart_status) * 0) + # --- NVMe ----------------------------------------------------------------- + # NVMe has neither an ATA attribute table nor SCSI error counters. Its health is + # in the SMART/Health Information log, which smartctl_exporter emits only when the + # interface is nvme. + # + # Each arm is reduced to {host, device} *before* the `or`, so the union + # deduplicates per drive rather than emitting one series per metric name -- the + # count is then "did anything fire", which is all smartctl:health_code asks. + - record: smartctl:nvme_failing:count + expr: | + ( + count by (host, device) (smartctl_device_critical_warning > 0) + or + count by (host, device) (smartctl_device_media_errors > 0) + or + count by (host, device) ( + smartctl_device_available_spare + < on(host, device) smartctl_device_available_spare_threshold + ) + ) + or (count by (host, device) (smartctl_device_smart_status) * 0) + + # percentage_used is the controller's own estimate of consumed write endurance, + # where 100 means the drive has reached its rated lifetime. Drives routinely run + # well past it, so this is a warning and not a failure. + - record: smartctl:nvme_caution:count + expr: | + count by (host, device) (smartctl_device_percentage_used >= 90) + or (count by (host, device) (smartctl_device_smart_status) * 0) + # --- Verdict -------------------------------------------------------------- # 0 good, 1 caution, 2 bad. The inner clamp_max on each group keeps two # simultaneous caution signals from adding up to a false "bad". @@ -69,22 +99,30 @@ groups: (max by (host, device) (smartctl_device_smart_status) == bool 0) + (smartctl:attr_failing:count > bool 0) + (smartctl:scsi_uncorrected:count > bool 0) + + (smartctl:nvme_failing:count > bool 0) , 1) + clamp_max( (smartctl:attr_caution:count > bool 0) + (smartctl:scsi_defects:count > bool 0) + + (smartctl:nvme_caution:count > bool 0) , 1) , 2) # --- Row labels ----------------------------------------------------------- # Which section of the SMART dashboard a drive belongs in. Keyed on metric # presence rather than the `protocol` label: only ATA drives report an attribute - # table, and only SCSI drives report a grown defect list. + # table, only SCSI drives report a grown defect list, and smartctl_exporter emits + # percentage_used only for nvme. The three are disjoint by construction, so the + # order of the `or` arms does not matter -- but a drive matching none of them gets + # no bus, drops out of the smartctl:row join, and silently loses its dashboard + # section, which is what NVMe used to do. - record: smartctl:bus expr: | label_replace(count by (host, device) (smartctl_device_attribute), "bus", "ata", "", "") or label_replace(count by (host, device) (smartctl_scsi_grown_defect_list), "bus", "sas", "", "") + or + label_replace(count by (host, device) (smartctl_device_percentage_used), "bus", "nvme", "", "") # The identity string from collectors/smart-selftest.py, made total so a drive # still gets a row if that collector has not written its file yet. `unless` (not |