diff options
Diffstat (limited to 'dashboards')
| -rw-r--r-- | dashboards/gen_smart.py | 584 | ||||
| -rw-r--r-- | dashboards/gen_zfs.py | 617 | ||||
| -rw-r--r-- | dashboards/panels.py | 291 | ||||
| -rw-r--r-- | dashboards/smart.json | 3844 | ||||
| -rw-r--r-- | dashboards/zfs-pools.json | 4372 |
5 files changed, 9708 insertions, 0 deletions
diff --git a/dashboards/gen_smart.py b/dashboards/gen_smart.py new file mode 100644 index 0000000..2cc0568 --- /dev/null +++ b/dashboards/gen_smart.py @@ -0,0 +1,584 @@ +#!/usr/bin/env python3 +"""Generate the "Drive Health (SMART)" Grafana dashboard. + +Run this and commit the result: + + python3 dashboards/gen_smart.py > dashboards/smart.json + +A CrystalDiskInfo-style view: a Good/Caution/Bad verdict per drive, the full raw SMART +attribute table, and self-test history. + +The fleet is mixed, so there are two repeating sections: + +* SATA drives expose the classic ATA attribute table (value / worst / thresh / raw). +* SAS drives expose none of it. Their health lives in SCSI error counters instead -- + grown defect list, uncorrected errors, corrected-error breakdown. + +Drives are routed into the right section by *metric presence* rather than by the +`protocol` label, which is more robust: only ATA drives have smartctl_device_attribute, +and only SCSI drives have smartctl_scsi_grown_defect_list. + +Metrics used: + +* smartctl_* -- prometheus-community/smartctl_exporter +* smartctl:health_code etc. -- recording rules in rules/smartctl.yml +* smart_selftest_* -- collectors/smart-selftest.py +""" + +import json + +from panels import (OK_BAD, dashboard, gp, query_var, row, stat, table, + timeseries) + +# Matches the verdict encoding in rules/smartctl.yml. +HEALTH_MAP = [{ + "type": "value", + "options": { + "0": {"text": "GOOD", "color": "green", "index": 0}, + "1": {"text": "CAUTION", "color": "orange", "index": 1}, + "2": {"text": "BAD", "color": "red", "index": 2}, + }, +}] + +HEALTH_THRESHOLDS = {"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "orange", "value": 1}, + {"color": "red", "value": 2}, +]} + +PASS_MAP = [{ + "type": "value", + "options": { + "0": {"text": "FAILED", "color": "red", "index": 0}, + "1": {"text": "passed", "color": "green", "index": 1}, + }, +}] + +ATA = 'device=~"$ata_device"' +SAS = 'device=~"$sas_device"' + +# Splits "sde · GOOD · 6.0 TB · SAS 7.2K · HGST HUS726060AL5210 · NAHPJS45" into a +# selectable value (the leading device name) and display text (the whole line). The +# groups nest rather than sit side by side, which is what lets text be a superset of +# value; matching on whitespace keeps it independent of the separator the collector +# chose. +ROW_REGEX = r"/^(?<text>(?<value>\S+)\s.*)$/" + + +def power_on(sel=""): + """Best available powered-on seconds per drive, in seconds. + + On ATA, smartctl reads power_on_time straight from attribute 9 and it is reliable. + On SAS there is no such attribute: smartctl sources power_on_time from the + Background Scan Results log page, whose "Accumulated power on time" counter is + maintained by the background-scan subsystem rather than being the drive's lifetime + meter. On the HGST SAS drives here it reads a few thousand hours while the drives' + own self-test logs record tens of thousands. + + The self-test log's lifetime-hours column is stamped by the drive at the moment + each test ran, so it can never exceed the true powered-on figure. Taking whichever + source is larger is therefore never an over-estimate, and recovers a sane number on + drives whose power-on counter under-counts or was reset. + """ + braces = f"{{{sel}}}" if sel else "" + return (f'max by (device) (smartctl_device_power_on_seconds{braces} ' + f'or (smart_selftest_last_lifetime_hours{braces} * 3600))') + + +def drive_info(title, sel, pos, label_cols, renames, desc=""): + """One-row identity card for a drive: what it is, not how it is doing. + + smartctl_device is an info metric whose value is always 1 and whose payload is + entirely in its labels, so the value column gets dropped. Rotation rate and + capacity are joined in as real numbers; the join is an outer one so an SSD, which + reports no rotation rate at all, still shows up with a blank RPM cell. + """ + return table( + title, [ + (f'smartctl_device{{{sel}}}', None), + (f'max by (device) (smartctl_device_rotation_rate{{{sel}}})', None), + (f'max by (device) (smartctl_device_capacity_bytes{{{sel}}})', None), + ], + pos, + keep="device|" + "|".join(label_cols) + "|Value.*", + renames={"__join__": "device", "device": "Device", + "Value #B": "RPM", "Value #C": "Capacity", **renames}, + exclude=["Value #A"], + order=["device", *label_cols, "Value #B", "Value #C"], + overrides=[ + {"matcher": {"id": "byName", "options": "Capacity"}, + "properties": [{"id": "unit", "value": "bytes"}]}, + {"matcher": {"id": "byName", "options": "RPM"}, + "properties": [{"id": "unit", "value": "rotrpm"}, + {"id": "noValue", "value": "SSD"}]}, + ], + desc=desc, + ) + + +panels = [] + +# ====================================================================================== +# Fleet Overview +# ====================================================================================== +panels.append(row("Fleet Overview", 0)) + +panels.append(stat( + "Healthy", [('count(smartctl:health_code == 0) or vector(0)', "drives")], + gp(0, 1, 4, 4), colormode="background", + thresholds={"mode": "absolute", "steps": [{"color": "green", "value": None}]}, + desc="Drives with no failing attribute, no SCSI uncorrected errors and a passing " + "SMART self-assessment.", +)) +panels.append(stat( + "Caution", [('count(smartctl:health_code == 1) or vector(0)', "drives")], + gp(4, 1, 4, 4), colormode="background", + thresholds={"mode": "absolute", "steps": [ + {"color": "green", "value": None}, {"color": "orange", "value": 1}]}, + desc="Reallocated / pending / uncorrectable sectors on SATA, or a non-empty grown " + "defect list on SAS. Not failing yet — worth watching the trend.", +)) +panels.append(stat( + "Failing", [('count(smartctl:health_code == 2) or vector(0)', "drives")], + gp(8, 1, 4, 4), colormode="background", thresholds=OK_BAD, + desc="SMART self-assessment failed, an attribute is at or below its threshold, or " + "the drive reported errors it could not correct. Replace these.", +)) +panels.append(stat( + "Drives", [('count(smartctl_device_smart_status)', "total")], + gp(12, 1, 4, 4), +)) +panels.append(stat( + "Hottest Drive", + [('max(smartctl_device_temperature{temperature_type="current"})', "max")], + gp(16, 1, 4, 4), unit="celsius", colormode="background", + thresholds={"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 45}, + {"color": "red", "value": 55}, + ]}, +)) +panels.append(stat( + "Longest Powered On", [(f'max({power_on()})', "max")], + gp(20, 1, 4, 4), unit="s", +)) + +panels.append(table( + "All Drives", [ + # group_left carries the descriptive labels from the info metric onto the + # verdict, so this one query supplies device, model, serial and protocol. + ('smartctl:health_code * on(device) group_left(model_name, serial_number, ' + 'protocol) smartctl_device', None), + ('max by (device) (smartctl_device_temperature{temperature_type="current"})', + None), + (power_on(), None), + ('max by (device) (smartctl_device_capacity_bytes)', None), + ], + gp(0, 5, 16, 10), keep="device|model_name|serial_number|protocol|Value.*", + renames={ + "__join__": "device", "device": "Device", "model_name": "Model", + "serial_number": "Serial", "protocol": "Bus", + "Value #A": "Health", "Value #B": "Temp", + "Value #C": "Powered On", "Value #D": "Capacity", + }, + order=["device", "model_name", "serial_number", "protocol", + "Value #A", "Value #B", "Value #C", "Value #D"], + sort_by="Health", + overrides=[ + {"matcher": {"id": "byName", "options": "Health"}, + "properties": [ + {"id": "mappings", "value": HEALTH_MAP}, + {"id": "thresholds", "value": HEALTH_THRESHOLDS}, + {"id": "custom.cellOptions", + "value": {"type": "color-background", "mode": "basic"}}, + ]}, + {"matcher": {"id": "byName", "options": "Temp"}, + "properties": [{"id": "unit", "value": "celsius"}]}, + {"matcher": {"id": "byName", "options": "Powered On"}, + "properties": [{"id": "unit", "value": "s"}]}, + {"matcher": {"id": "byName", "options": "Capacity"}, + "properties": [{"id": "unit", "value": "bytes"}]}, + ], + desc="Every drive, worst first. Health is computed by rules/smartctl.yml.", +)) + +panels.append(timeseries( + "Temperatures", + [('smartctl_device_temperature{temperature_type="current"}', "{{device}}")], + gp(16, 5, 8, 10), unit="celsius", + desc="Sustained operation above ~45C measurably shortens drive life.", +)) + +# ====================================================================================== +# SATA — repeats once per ATA drive +# ====================================================================================== +panels.append(row("${ata_device:text}", 15, repeat="ata_device")) + +panels.append(stat( + "Health", [(f'smartctl:health_code{{{ATA}}}', "{{device}}")], + gp(0, 16, 6, 4), mappings=HEALTH_MAP, thresholds=HEALTH_THRESHOLDS, + colormode="background", +)) +panels.append(stat( + "Temperature", + [(f'smartctl_device_temperature{{{ATA}, temperature_type="current"}}', "temp")], + gp(6, 16, 3, 4), unit="celsius", + thresholds={"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 45}, + {"color": "red", "value": 55}, + ]}, +)) +panels.append(stat( + "Powered On", [(power_on(ATA), "hours")], + gp(9, 16, 3, 4), unit="s", +)) +panels.append(stat( + "Capacity", [(f'smartctl_device_capacity_bytes{{{ATA}}}', "bytes")], + gp(12, 16, 3, 4), unit="bytes", +)) +panels.append(stat( + "Power Cycles", [(f'smartctl_device_power_cycle_count{{{ATA}}}', "cycles")], + gp(15, 16, 3, 4), +)) +panels.append(stat( + "Reallocated Sectors", + [(f'smartctl_device_attribute{{{ATA}, attribute_id="5", ' + f'attribute_value_type="raw"}}', "sectors")], + gp(18, 16, 3, 4), thresholds=OK_BAD, colormode="background", + desc="Attribute 5 raw. Sectors the drive has already retired. Any growth here is " + "the single most predictive sign of impending failure.", +)) +panels.append(stat( + "Pending Sectors", + [(f'smartctl_device_attribute{{{ATA}, attribute_id="197", ' + f'attribute_value_type="raw"}}', "sectors")], + gp(21, 16, 3, 4), thresholds=OK_BAD, colormode="background", + desc="Attribute 197 raw. Sectors that failed to read and are waiting to be " + "reallocated on next write. Often precedes reallocations.", +)) + +panels.append(drive_info( + "Drive — $ata_device", ATA, gp(0, 20, 24, 4), + label_cols=["model_family", "model_name", "serial_number", "firmware_version", + "interface", "sata_version", "form_factor"], + renames={"model_family": "Family", "model_name": "Model", + "serial_number": "Serial", "firmware_version": "Firmware", + "interface": "Interface", "sata_version": "SATA", "form_factor": "Form"}, + desc="Identity of this drive. RPM shows SSD when the drive reports no rotation " + "rate.", +)) + +panels.append(table( + "SMART Attributes — $ata_device", [ + # Only this query keeps the descriptive labels; the rest are reduced to the + # join key so nothing collides in joinByField. + (f'smartctl_device_attribute{{{ATA}, attribute_value_type="value"}}', None), + (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, ' + f'attribute_value_type="worst"}})', None), + (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, ' + f'attribute_value_type="thresh"}})', None), + (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, ' + f'attribute_value_type="raw"}})', None), + # How much headroom is left before this attribute trips its own failure + # threshold. Zero or below is what SMART calls a failure. + (f'max by (attribute_id) (smartctl_device_attribute{{{ATA}, ' + f'attribute_value_type="value"}} - on(device, attribute_id) ' + f'smartctl_device_attribute{{{ATA}, attribute_value_type="thresh"}})', None), + ], + gp(0, 24, 24, 12), keep="attribute_id|attribute_name|Value.*", + renames={ + "__join__": "attribute_id", "attribute_id": "ID", + "attribute_name": "Attribute", + "Value #A": "Current", "Value #B": "Worst", "Value #C": "Threshold", + "Value #D": "Raw", "Value #E": "Margin", + }, + order=["attribute_id", "attribute_name", + "Value #A", "Value #B", "Value #C", "Value #D", "Value #E"], + sort_by="Attribute", sort_desc=False, + overrides=[ + # Margin is the honest place to highlight trouble: a blanket threshold on Raw + # would flag Power_On_Hours, which is enormous and perfectly healthy. + {"matcher": {"id": "byName", "options": "Margin"}, + "properties": [ + {"id": "thresholds", "value": {"mode": "absolute", "steps": [ + {"color": "red", "value": None}, + {"color": "orange", "value": 1}, + {"color": "yellow", "value": 10}, + {"color": "green", "value": 25}, + ]}}, + {"id": "custom.cellOptions", + "value": {"type": "color-background", "mode": "basic"}}, + ]}, + ], + desc="Every attribute the drive reports, with its raw value. Current/Worst are " + "normalised 0-253 scores (higher is better); Threshold is where the vendor " + "declares failure; Margin is Current minus Threshold. ID sorts as text, so " + "the table is ordered by attribute name.", +)) + +# ====================================================================================== +# SAS — repeats once per SCSI drive +# ====================================================================================== +panels.append(row("${sas_device:text}", 36, repeat="sas_device")) + +panels.append(stat( + "Health", [(f'smartctl:health_code{{{SAS}}}', "{{device}}")], + gp(0, 37, 6, 4), mappings=HEALTH_MAP, thresholds=HEALTH_THRESHOLDS, + colormode="background", +)) +panels.append(stat( + "Temperature", + [(f'smartctl_device_temperature{{{SAS}, temperature_type="current"}}', "temp")], + gp(6, 37, 3, 4), unit="celsius", + thresholds={"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 45}, + {"color": "red", "value": 55}, + ]}, +)) +panels.append(stat( + "Powered On", [(power_on(SAS), "hours")], + gp(9, 37, 3, 4), unit="s", + desc="SAS drives have no ATA attribute 9, so smartctl reports power_on_time from " + "the Background Scan Results log page — a counter the background-scan " + "subsystem maintains, not the drive's lifetime meter, and one that commonly " + "reads far too low. This tile takes the larger of that counter and the " + "lifetime-hours stamp in the drive's own self-test log, which is why it can " + "exceed what `smartctl -x` prints as 'Accumulated power on time'.", +)) +panels.append(stat( + "Capacity", [(f'smartctl_device_capacity_bytes{{{SAS}}}', "bytes")], + gp(12, 37, 3, 4), unit="bytes", +)) +panels.append(stat( + "Link Speed", + [(f'smartctl_device_interface_speed{{{SAS}, speed_type="current"}} * 1000', "bps")], + gp(15, 37, 3, 4), unit="bps", + desc="Negotiated SAS link rate. A drive that has fallen back to a lower rate is " + "worth investigating — usually a cable or expander.", +)) +panels.append(stat( + "Grown Defects", [(f'smartctl_scsi_grown_defect_list{{{SAS}}}', "defects")], + gp(18, 37, 3, 4), thresholds=OK_BAD, colormode="background", + desc="Sectors reallocated since the drive left the factory — the SAS equivalent " + "of ATA attribute 5. A stable non-zero count is tolerable; growth is not.", +)) +panels.append(stat( + "Uncorrected Errors", + [(f'smartctl_read_total_uncorrected_errors{{{SAS}}} + ' + f'smartctl_write_total_uncorrected_errors{{{SAS}}}', "errors")], + gp(21, 37, 3, 4), thresholds=OK_BAD, colormode="background", + desc="Errors the drive could not correct with ECC or retries, which means data " + "was lost. Should be zero.", +)) + +panels.append(drive_info( + "Drive — $sas_device", SAS, gp(0, 41, 24, 4), + label_cols=["scsi_vendor", "model_name", "serial_number", "scsi_revision", + "scsi_version", "protocol", "form_factor"], + renames={"scsi_vendor": "Vendor", "model_name": "Model", + "serial_number": "Serial", "scsi_revision": "Revision", + "scsi_version": "SCSI", "protocol": "Bus", "form_factor": "Form"}, + desc="Identity of this drive. SAS drives leave firmware_version empty and report " + "their revision in scsi_revision instead.", +)) + +panels.append(timeseries( + "Corrected Errors — $sas_device", [ + (f'smartctl_read_errors_corrected_by_eccfast{{{SAS}}}', "read ECC fast"), + (f'smartctl_read_errors_corrected_by_eccdelayed{{{SAS}}}', "read ECC delayed"), + (f'smartctl_read_errors_corrected_by_rereads_rewrites{{{SAS}}}', "read retries"), + (f'smartctl_write_errors_corrected_by_eccfast{{{SAS}}}', "write ECC fast"), + (f'smartctl_write_errors_corrected_by_eccdelayed{{{SAS}}}', + "write ECC delayed"), + (f'smartctl_write_errors_corrected_by_rereads_rewrites{{{SAS}}}', + "write retries"), + ], + gp(0, 45, 12, 8), minval=0, + desc="Errors the drive fixed on its own. ECC-fast corrections are routine and " + "grow constantly. Delayed corrections and especially rewrites/retries " + "climbing is the signal worth watching.", +)) +panels.append(timeseries( + "Defects & Uncorrected — $sas_device", [ + (f'smartctl_scsi_grown_defect_list{{{SAS}}}', "grown defects"), + (f'smartctl_read_total_uncorrected_errors{{{SAS}}}', "read uncorrected"), + (f'smartctl_write_total_uncorrected_errors{{{SAS}}}', "write uncorrected"), + ], + gp(12, 45, 12, 8), minval=0, + custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, + desc="Flat lines are good. Any upward step here is a real event.", +)) + +# ====================================================================================== +# Self-tests +# ====================================================================================== +panels.append(row("Self-Tests", 53)) + +panels.append(stat( + "Drives Never Tested", + [('count(smartctl_device_smart_status) - ' + 'count(count by (device) (smart_selftest_last_lifetime_hours)) or ' + 'count(smartctl_device_smart_status)', "drives")], + gp(0, 54, 4, 5), thresholds=OK_BAD, colormode="background", + desc="Drives with no self-test log entries at all. smartd only runs tests if its " + "config includes an -s schedule; merely enabling the service does not.", +)) +panels.append(stat( + "Failed Self-Tests", + [('count(smart_selftest_last_passed == 0) or vector(0)', "tests")], + gp(4, 54, 4, 5), thresholds=OK_BAD, colormode="background", +)) +panels.append(stat( + "Oldest Long Test", + [(f'max({power_on()} / 3600 - on(device) group_right() ' + f'smart_selftest_last_lifetime_hours{{test_type="extended"}})', "hours ago")], + gp(8, 54, 4, 5), unit="h", + thresholds={"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 24 * 14}, + {"color": "red", "value": 24 * 45}, + ]}, + no_value="never run", + desc="Powered-on hours since the least recently tested drive last completed an " + "extended (long) self-test. Shows 'never run' when no drive has ever " + "completed one — which is the default state, since smartd schedules no tests " + "unless its config carries an -s directive.", +)) + +def _keyed(expr): + """Reduce a series to just the synthetic join key. + + joinByField takes a single field, but a row here is identified by device *and* + test_type together. Joining on either one alone silently mismatches rows across + drives, so build one composite label and join on that instead. + """ + return (f'max by (drive_test) (label_join({expr}, "drive_test", "/", ' + f'"device", "test_type"))') + + +panels.append(table( + "Self-Test History", [ + # Only this query keeps device and test_type as their own columns. + ('label_join(smart_selftest_last_passed, "drive_test", "/", "device", ' + '"test_type")', None), + (_keyed('smart_selftest_last_lifetime_hours'), None), + (_keyed('smart_selftest_count'), None), + # Powered-on hours elapsed since that test ran. group_right() keeps the + # right-hand side's labels, so device and test_type survive the subtraction. + (_keyed(power_on() + ' / 3600 - on(device) group_right() ' + 'smart_selftest_last_lifetime_hours'), None), + ], + gp(12, 54, 12, 10), keep="drive_test|device|test_type|Value.*", + renames={ + "__join__": "drive_test", "device": "Device", "test_type": "Test", + "Value #A": "Last Result", "Value #B": "At Hours", + "Value #C": "Logged", "Value #D": "Hours Ago", + }, + exclude=["drive_test"], + order=["device", "test_type", "Value #A", "Value #B", "Value #C", "Value #D"], + sort_by="Device", sort_desc=False, + overrides=[ + {"matcher": {"id": "byName", "options": "Last Result"}, + "properties": [ + {"id": "mappings", "value": PASS_MAP}, + {"id": "custom.cellOptions", + "value": {"type": "color-background", "mode": "basic"}}, + ]}, + ], + desc="Short and extended self-test results per drive, from " + "collectors/smart-selftest.py. 'At Hours' is the drive's powered-on hour " + "count when the test ran, straight from the log. 'Hours Ago' subtracts it " + "from the drive's current powered-on hours; on SAS that figure already falls " + "back to this log, so 0 means the newest log entry is also the highest hour " + "count the drive has ever reported — not that the test ran just now.", +)) + +# ====================================================================================== +# Collapsed: trends and diagnostics +# ====================================================================================== +panels.append(row("Attribute Trends", 64, collapsed=True, children=[ + timeseries( + "Reallocated Sectors (all SATA drives)", + [('smartctl_device_attribute{attribute_id="5", attribute_value_type="raw"}', + "{{device}}")], + gp(0, 65, 8, 7), minval=0, + custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, + ), + timeseries( + "Pending & Uncorrectable Sectors", + [('smartctl_device_attribute{attribute_id=~"197|198", ' + 'attribute_value_type="raw"}', "{{device}} {{attribute_name}}")], + gp(8, 65, 8, 7), minval=0, + custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, + ), + timeseries( + "CRC Errors (cabling)", + [('smartctl_device_attribute{attribute_id="199", attribute_value_type="raw"}', + "{{device}}")], + gp(16, 65, 8, 7), minval=0, + custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, + desc="Attribute 199 counts errors on the SATA link rather than the platters. " + "Growth here usually means a cable or backplane, not a dying disk.", + ), + timeseries( + "Data Written / Read (SAS)", [ + ('smartctl_device_bytes_written', "{{device}} written"), + ('smartctl_device_bytes_read', "{{device}} read"), + ], + gp(0, 72, 12, 7), unit="bytes", minval=0, + ), + timeseries( + "Health Code History", + [('smartctl:health_code', "{{device}}")], + gp(12, 72, 12, 7), mappings=HEALTH_MAP, minval=0, maxval=2, + custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"}, + ), +])) + +panels.append(row("Collector Health", 65, collapsed=True, children=[ + timeseries( + "smartctl Exit Status", + [('smartctl_device_smartctl_exit_status', "{{device}}")], + gp(0, 66, 8, 7), minval=0, + desc="Non-zero means smartctl itself had trouble reading the device. Bit 0 is " + "a command-line problem, bit 1 a device-open failure, bit 3 a failing " + "drive.", + ), + timeseries( + "Self-Test Collector Duration", + [('smart_selftest_collection_duration_seconds', "collection")], + gp(8, 66, 8, 7), unit="s", minval=0, + ), + timeseries( + "Devices Seen", [ + ('smartctl_devices', "exporter"), + ('smart_selftest_devices', "self-test collector"), + ], + gp(16, 66, 8, 7), minval=0, + desc="These should agree. A divergence means one of the two cannot see some " + "drives.", + ), +])) + +print(json.dumps(dashboard( + title="Drive Health (SMART)", + uid="drive-health-smart", + description="Per-drive SMART health in the style of CrystalDiskInfo: a " + "Good/Caution/Bad verdict, full raw attribute tables for SATA, SCSI " + "error counters for SAS, and self-test history.", + panels=panels, + variables=[ + # smartctl:row is one rendered line per drive, "sde · GOOD · 6.0 TB · ...". + # The nested capture groups keep the *value* a bare device name, so every + # `device=~"$ata_device"` selector below is unaffected, while the *text* the UI + # shows -- and that ${ata_device:text} puts in the row title -- is the whole + # line. That is what makes a collapsed row identifiable without opening it. + query_var("ata_device", 'label_values(smartctl:row{bus="ata"}, row)', + label="SATA drive", regex=ROW_REGEX), + query_var("sas_device", 'label_values(smartctl:row{bus="sas"}, row)', + label="SAS drive", regex=ROW_REGEX), + ], + tags=["smart", "storage", "hardware"], +), indent=2)) diff --git a/dashboards/gen_zfs.py b/dashboards/gen_zfs.py new file mode 100644 index 0000000..4edb38b --- /dev/null +++ b/dashboards/gen_zfs.py @@ -0,0 +1,617 @@ +#!/usr/bin/env python3 +"""Generate the "ZFS Pools" Grafana dashboard. + +Run this and commit the result: + + python3 dashboards/gen_zfs.py > dashboards/zfs-pools.json + +Hand-writing ~2000 lines of Grafana panel JSON invites subtle inconsistencies, so +panels are built from a handful of helpers instead -- every timeseries goes through one +code path and is therefore structurally identical by construction. + +The queries here are deliberately limited to metrics we actually produce: + +* zfs_pool_* / zfs_dataset_* / zfs_snapshot_* -- collectors/zfs-metrics.py +* node_zfs_arc_* / node_zfs_zpool_dataset_* -- node_exporter's zfs collector +* node_systemd_* -- node_exporter's systemd collector + +If you add a panel, add the metric to the collector first and keep this list honest. +""" + +import json + +from panels import (DS, OK_BAD, bargauge, gauge, gp, row, stat, table, timeseries) + + +STATE_MAP = [{ + "type": "value", + "options": { + "0": {"text": "ONLINE", "color": "green", "index": 0}, + "1": {"text": "DEGRADED", "color": "orange", "index": 1}, + "2": {"text": "FAULTED", "color": "red", "index": 2}, + "3": {"text": "OFFLINE", "color": "text", "index": 3}, + "4": {"text": "UNAVAIL", "color": "red", "index": 4}, + "5": {"text": "REMOVED", "color": "purple", "index": 5}, + "6": {"text": "SUSPENDED", "color": "red", "index": 6}, + "7": {"text": "AVAIL", "color": "blue", "index": 7}, + "8": {"text": "UNKNOWN", "color": "text", "index": 8}, + }, +}] + +BOOL_MAP = [{ + "type": "value", + "options": { + "0": {"text": "idle", "color": "green", "index": 0}, + "1": {"text": "RUNNING", "color": "blue", "index": 1}, + }, +}] + +POOL = 'pool=~"$pool"' +panels = [] + +# ====================================================================================== +# Pool Overview +# ====================================================================================== +panels.append(row("Pool Overview", 0)) + +panels.append(stat( + "Pool Health", [(f'zfs_pool_health_code{{{POOL}}}', "{{pool}}")], + gp(0, 1, 4, 4), mappings=STATE_MAP, colormode="background", + desc="Health as reported by `zpool list -o health`.", +)) +panels.append(stat( + "Data Errors", [(f'sum(zfs_pool_data_errors{{{POOL}}}) or vector(0)', "errors")], + gp(4, 1, 4, 4), thresholds=OK_BAD, colormode="background", + desc="Permanent, unrepairable errors from `zpool status`. Anything above zero " + "means data has been lost and the affected files need restoring.", +)) +panels.append(stat( + "vdev Errors", + [(f'sum(zfs_pool_vdev_errors_total{{{POOL}}}) or vector(0)', "errors")], + gp(8, 1, 4, 4), thresholds=OK_BAD, colormode="background", + desc="Total read + write + checksum errors across all vdevs. Cleared by " + "`zpool clear`.", +)) +panels.append(stat( + "Last Scrub", + [(f'time() - max(zfs_pool_last_scrub_end_timestamp_seconds{{{POOL}}})', "ago")], + gp(12, 1, 4, 4), unit="s", colormode="background", + thresholds={"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 35 * 86400}, + {"color": "red", "value": 70 * 86400}, + ]}, + desc="Time since the most recent completed scrub. Turns amber past ~5 weeks, " + "which is a monthly scrub schedule having missed a run.", +)) +panels.append(stat( + "Scrub / Resilver", [ + (f'max(zfs_pool_scrub_in_progress{{{POOL}}})', "scrub"), + (f'max(zfs_pool_resilver_in_progress{{{POOL}}})', "resilver"), + ], + gp(16, 1, 4, 4), mappings=BOOL_MAP, colormode="background", +)) +panels.append(stat( + "Locked Datasets", + [(f'count(zfs_dataset_encrypted{{{POOL}}} == 1 and ' + f'zfs_dataset_key_available{{{POOL}}} == 0) or vector(0)', "locked")], + gp(20, 1, 4, 4), thresholds=OK_BAD, colormode="background", + desc="Encrypted datasets whose key is not loaded. These are unreadable and are " + "silently skipped by anything that walks the filesystem, including backups.", +)) + +panels.append(gauge( + "Space Used", [(f'zfs_pool_capacity_ratio{{{POOL}}}', "{{pool}}")], + gp(0, 5, 5, 4), + desc="How full the pool is: allocated / total. ZFS write performance degrades " + "sharply past ~80%, and the pool becomes effectively read-only near 100%. " + "The byte figures behind this are in 'Capacity by Pool' below.", +)) +panels.append(gauge( + "Fragmentation", [(f'zfs_pool_fragmentation_ratio{{{POOL}}}', "{{pool}}")], + gp(5, 5, 5, 4), + thresholds={"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 0.5}, + {"color": "red", "value": 0.75}, + ]}, + desc="Free-space fragmentation.", +)) +panels.append(stat( + "Free Space", [(f'zfs_pool_free_bytes{{{POOL}}}', "{{pool}}")], + gp(10, 5, 5, 4), unit="bytes", +)) +panels.append(stat( + "Snapshots Overdue", + [(f'count((time() - zfs_snapshot_newest_timestamp_seconds{{{POOL}}}) > on(label) ' + f'group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)', "overdue")], + gp(15, 5, 4, 4), thresholds=OK_BAD, colormode="background", + desc="Dataset/label pairs whose newest snapshot is more than two intervals old — " + "i.e. the auto-snapshot timer has missed at least one run.", +)) +panels.append(stat( + "Retention Depth", + [(f'time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{{{POOL}}})', "oldest")], + gp(19, 5, 5, 4), unit="s", + desc="Age of the oldest snapshot on the pool — how far back you can actually " + "restore from.", +)) + +panels.append(timeseries( + "Pool Space Used / Free", [ + (f'zfs_pool_allocated_bytes{{{POOL}}}', "{{pool}} used"), + (f'zfs_pool_free_bytes{{{POOL}}}', "{{pool}} free"), + ], + gp(0, 9, 12, 8), unit="bytes", stack=True, minval=0, + desc="Stacked, so the top of each pool's band is its total size and the split " + "between the bands is the usage fraction the gauge above shows.", +)) +panels.append(timeseries( + "Fragmentation", + [(f'zfs_pool_fragmentation_ratio{{{POOL}}}', "{{pool}}")], + gp(12, 9, 12, 8), unit="percentunit", minval=0, maxval=1, + desc="Free-space fragmentation — how scattered the pool's remaining free space " + "is, not how scattered the data is. It rises as a pool fills and stays high " + "afterwards; only rewriting data (send/recv, or deleting a lot) brings it " + "down. Kept on its own axis because it is unrelated to space used, and " + "plotting the two together invites reading one as the other.", +)) + +panels.append(table( + "Capacity by Pool", [ + (f'zfs_pool_size_bytes{{{POOL}}}', None), + (f'max by (pool) (zfs_pool_allocated_bytes{{{POOL}}})', None), + (f'max by (pool) (zfs_pool_free_bytes{{{POOL}}})', None), + (f'max by (pool) (zfs_pool_capacity_ratio{{{POOL}}})', None), + (f'max by (pool) (zfs_pool_fragmentation_ratio{{{POOL}}})', None), + (f'max by (pool) (zfs_pool_snapshot_used_bytes{{{POOL}}})', None), + (f'max by (pool) (zfs_pool_dedup_ratio{{{POOL}}})', None), + ], + gp(0, 17, 24, 8), keep="pool|Value.*", + renames={ + "__join__": "pool", "pool": "Pool", + "Value #A": "Size", "Value #B": "Used", "Value #C": "Free", + "Value #D": "Used %", "Value #E": "Fragmentation", + "Value #F": "Snapshots", "Value #G": "Dedup", + }, + order=["pool", "Value #A", "Value #B", "Value #C", "Value #D", "Value #E", + "Value #F", "Value #G"], + sort_by="Used %", + overrides=[ + # Anchored: byRegexp is a substring match, so a bare "Used" would also + # capture "Used %" and fight the percent override below. + {"matcher": {"id": "byRegexp", "options": "^(Size|Used|Free|Snapshots)$"}, + "properties": [{"id": "unit", "value": "bytes"}]}, + {"matcher": {"id": "byName", "options": "Used %"}, + "properties": [ + {"id": "unit", "value": "percentunit"}, + {"id": "max", "value": 1}, + {"id": "thresholds", "value": {"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 0.8}, + {"color": "red", "value": 0.9}, + ]}}, + {"id": "custom.cellOptions", + "value": {"type": "gauge", "mode": "gradient"}}, + ]}, + {"matcher": {"id": "byName", "options": "Fragmentation"}, + "properties": [{"id": "unit", "value": "percentunit"}]}, + ], + desc="Per-pool space in bytes. 'Size' is the raw pool size zpool reports, which " + "for raidz is before parity overhead — so 'Free' is the number to trust for " + "how much you can still write. 'Snapshots' is space held only by snapshots.", +)) + +# ====================================================================================== +# Integrity & Errors +# ====================================================================================== +panels.append(row("Integrity & Errors", 25)) + +panels.append(table( + "vdev State", [(f'zfs_pool_vdev_state_code{{{POOL}}}', None)], + gp(0, 26, 12, 9), keep="pool|vdev|Value", + renames={"pool": "Pool", "vdev": "vdev", "Value": "State"}, + mappings=STATE_MAP, + overrides=[{ + "matcher": {"id": "byName", "options": "State"}, + "properties": [{"id": "custom.cellOptions", + "value": {"type": "color-background", "mode": "basic"}}], + }], + desc="Every vdev and leaf device in the selected pool(s).", +)) +panels.append(timeseries( + "vdev Error Counters", + [(f'zfs_pool_vdev_errors_total{{{POOL}}}', "{{pool}}/{{vdev}} {{type}}")], + gp(12, 26, 12, 9), minval=0, + custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, + desc="Cumulative per-vdev errors. A counter that climbs is a disk on its way out.", +)) + +panels.append(table( + "Scrub & Resilver History", [ + (f'zfs_pool_last_scrub_end_timestamp_seconds{{{POOL}}} * 1000', None), + (f'zfs_pool_last_scrub_duration_seconds{{{POOL}}}', None), + (f'zfs_pool_last_scrub_errors{{{POOL}}}', None), + (f'zfs_pool_last_scrub_repaired_bytes{{{POOL}}}', None), + # zpool status reports one scan per pool, so a pool whose most recent scan was + # a resilver has no scrub record at all and would otherwise vanish from this + # table. The outer join keeps it, with only this column filled. + (f'zfs_pool_last_resilver_end_timestamp_seconds{{{POOL}}} * 1000', None), + ], + gp(0, 35, 12, 7), keep="pool|Value.*", + renames={ + "__join__": "pool", + "pool": "Pool", + "Value #A": "Scrub Finished", "Value #B": "Duration", + "Value #C": "Errors", "Value #D": "Repaired", + "Value #E": "Resilver Finished", + }, + overrides=[ + {"matcher": {"id": "byName", "options": "Scrub Finished"}, + "properties": [{"id": "unit", "value": "dateTimeAsIso"}]}, + {"matcher": {"id": "byName", "options": "Resilver Finished"}, + "properties": [{"id": "unit", "value": "dateTimeAsIso"}]}, + {"matcher": {"id": "byName", "options": "Duration"}, + "properties": [{"id": "unit", "value": "s"}]}, + {"matcher": {"id": "byName", "options": "Repaired"}, + "properties": [{"id": "unit", "value": "bytes"}]}, + {"matcher": {"id": "byName", "options": "Errors"}, + "properties": [ + {"id": "custom.cellOptions", + "value": {"type": "color-background", "mode": "basic"}}, + {"id": "thresholds", "value": OK_BAD}, + ]}, + ], + desc="Result of the last completed scrub per pool. Errors here are blocks that " + "could not be repaired from redundancy.", +)) +panels.append(timeseries( + "Pool Data Errors", [(f'zfs_pool_data_errors{{{POOL}}}', "{{pool}}")], + gp(12, 35, 12, 7), minval=0, + custom={"lineWidth": 2, "fillOpacity": 10, "showPoints": "auto"}, + desc="Permanent data errors over time.", +)) + +# ====================================================================================== +# Snapshots & Retention +# ====================================================================================== +panels.append(row("Snapshots & Retention", 42)) + +panels.append(bargauge( + "Time Since Last Snapshot", + [(f'time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{{{POOL}}})', + "{{label}}")], + gp(0, 43, 8, 8), unit="s", + thresholds={"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 2 * 3600}, + {"color": "red", "value": 2 * 86400}, + ]}, + desc="Age of the newest snapshot of each label. Compare against the label's own " + "cadence — 'frequent' is every 15 minutes, 'monthly' every month, so the " + "colour thresholds here are only a coarse hint.", +)) +panels.append(bargauge( + "Snapshots vs Retention Target", + [(f'min by (label) (zfs_snapshot_count{{{POOL}}} / on(label) group_left() ' + f'zfs_snapshot_retention_target)', "{{label}}")], + gp(8, 43, 8, 8), unit="percentunit", maxv=1.2, + thresholds={"mode": "absolute", "steps": [ + {"color": "red", "value": None}, + {"color": "yellow", "value": 0.5}, + {"color": "green", "value": 0.9}, + ]}, + desc="Worst dataset's snapshot count as a fraction of the configured " + "services.zfs.autoSnapshot retention. 100% means the full set is on disk; a " + "freshly-created dataset legitimately sits below target until it fills up.", +)) +panels.append(bargauge( + "Oldest Snapshot", + [(f'time() - min by (label) ' + f'(zfs_pool_snapshot_oldest_timestamp_seconds{{{POOL}}})', "{{label}}")], + gp(16, 43, 8, 8), unit="s", + desc="How far back each label reaches.", +)) + +panels.append(timeseries( + "Snapshot Count by Label", + [(f'sum by (label) (zfs_snapshot_count{{{POOL}}})', "{{label}}")], + gp(0, 51, 12, 8), minval=0, + desc="Total snapshots per label. A steady sawtooth is healthy rotation; a " + "monotonic decline means snapshots are expiring but not being replaced.", +)) +panels.append(timeseries( + "Snapshot Space by Label", + [(f'sum by (label) (zfs_pool_snapshot_used_bytes{{{POOL}}})', "{{label}}")], + gp(12, 51, 12, 8), unit="bytes", stack=True, minval=0, + desc="Space held down by snapshots — the cost of your retention policy.", +)) + +panels.append(stat( + "Snapshot / Scrub Job Failures", + [('sum(node_systemd_unit_state{name=~"zfs-(snapshot|scrub).*",state="failed"}) ' + 'or vector(0)', "failed units")], + gp(0, 59, 6, 6), thresholds=OK_BAD, colormode="background", + desc="systemd units for the auto-snapshot and auto-scrub timers currently in a " + "failed state.", +)) +panels.append(bargauge( + "Time Since Timer Last Fired", + [('time() - node_systemd_timer_last_trigger_seconds{name=~"zfs-.*"}', "{{name}}")], + gp(6, 59, 18, 6), unit="s", + desc="When each zfs-* systemd timer last triggered. This is the job side of the " + "picture; the snapshot ages above are the result side.", +)) + +# ====================================================================================== +# Datasets +# ====================================================================================== +panels.append(row("Datasets", 65)) + +panels.append(timeseries( + "Dataset Usage Over Time", + [(f'zfs_dataset_used_bytes{{{POOL}}}', "{{dataset}}")], + gp(0, 66, 24, 9), unit="bytes", minval=0, + custom={"lineWidth": 2, "fillOpacity": 0, "showPoints": "auto"}, + desc="Space each dataset is responsible for, including its snapshots and " + "descendants — so a parent's line contains its children's. Unstacked and " + "unfilled so a single dataset's growth is readable against the others rather " + "than riding on top of them. The legend carries last and max per dataset; the " + "instant breakdown is in 'Capacity by Dataset' below.", +)) + +panels.append(table( + "Capacity by Dataset", [ + # Only this query keeps `pool` as its own column; the rest are reduced to the + # join key so joinByField has nothing to collide on. `dataset` is the full + # path ("ingens/media"), so it is unique on its own. + (f'zfs_dataset_used_bytes{{{POOL}}}', None), + (f'max by (dataset) (zfs_dataset_used_by_dataset_bytes{{{POOL}}})', None), + (f'max by (dataset) (zfs_dataset_used_by_snapshots_bytes{{{POOL}}})', None), + (f'max by (dataset) (zfs_dataset_referenced_bytes{{{POOL}}})', None), + (f'max by (dataset) (zfs_dataset_logical_used_bytes{{{POOL}}})', None), + (f'max by (dataset) (zfs_dataset_available_bytes{{{POOL}}})', None), + (f'max by (dataset) (zfs_dataset_compression_ratio{{{POOL}}})', None), + # Quota is 0 when unset, which would read as "no space allowed"; blank it. + (f'max by (dataset) (zfs_dataset_quota_bytes{{{POOL}}} > 0)', None), + ], + gp(0, 75, 24, 8), keep="pool|dataset|Value.*", + renames={ + "__join__": "dataset", "dataset": "Dataset", "pool": "Pool", + "Value #A": "Used", "Value #B": "Own Data", "Value #C": "Snapshots", + "Value #D": "Referenced", "Value #E": "Logical", "Value #F": "Available", + "Value #G": "Ratio", "Value #H": "Quota", + }, + order=["pool", "dataset", "Value #A", "Value #B", "Value #C", "Value #D", + "Value #E", "Value #F", "Value #G", "Value #H"], + sort_by="Used", + overrides=[ + {"matcher": {"id": "byRegexp", + "options": "^(Used|Own Data|Snapshots|Referenced|Logical|" + "Available|Quota)$"}, + "properties": [{"id": "unit", "value": "bytes"}]}, + {"matcher": {"id": "byName", "options": "Quota"}, + "properties": [{"id": "noValue", "value": "none"}]}, + {"matcher": {"id": "byName", "options": "Ratio"}, + "properties": [{"id": "decimals", "value": 2}]}, + ], + desc="Per-dataset space, separate from the per-pool figures above. 'Used' is the " + "total a dataset is responsible for; 'Own Data' and 'Snapshots' are the two " + "halves of that. 'Logical' is the size before compression, so Logical / " + "Referenced is where the compression ratio comes from. 'Available' is shared " + "with every sibling dataset unless a quota says otherwise.", +)) + +panels.append(table( + "Compression", + [(f'zfs_dataset_compression_ratio{{{POOL}}} * on(pool, dataset) ' + f'group_left(compression) zfs_dataset_compression{{{POOL}}}', None)], + gp(0, 83, 12, 8), keep="pool|dataset|compression|Value", + order=["pool", "dataset", "compression", "Value"], + renames={"pool": "Pool", "dataset": "Dataset", + "compression": "Algorithm", "Value": "Ratio"}, + sort_by="Ratio", + desc="group_left() attaches the algorithm label from the info metric onto the " + "ratio, so this needs only one query and no join transformation.", +)) +panels.append(table( + "Encryption", + [(f'zfs_dataset_keystatus_code{{{POOL}}} * on(pool, dataset) ' + f'group_left(encryption) zfs_dataset_encryption{{{POOL}}}', None)], + gp(12, 83, 12, 8), keep="pool|dataset|encryption|Value", + order=["pool", "dataset", "encryption", "Value"], + renames={"pool": "Pool", "dataset": "Dataset", + "encryption": "Suite", "Value": "Key"}, + mappings=[{"type": "value", "options": { + "0": {"text": "—", "color": "text", "index": 0}, + "1": {"text": "unlocked", "color": "green", "index": 1}, + "2": {"text": "LOCKED", "color": "red", "index": 2}, + "3": {"text": "unknown", "color": "orange", "index": 3}, + }}], + overrides=[{ + "matcher": {"id": "byName", "options": "Key"}, + "properties": [{"id": "custom.cellOptions", + "value": {"type": "color-background", "mode": "basic"}}], + }], + desc="Encryption suite per dataset and the state of its key. Datasets that are " + "not encrypted show suite 'off' and a dash — only an encrypted dataset can " + "be locked.", +)) + +# ====================================================================================== +# Performance +# ====================================================================================== +panels.append(row("Performance", 91)) + +panels.append(timeseries( + "Dataset Throughput", [ + (f'rate(node_zfs_zpool_dataset_nread{{zpool=~"$pool"}}[$__rate_interval])', + "{{zpool}}/{{dataset}} read"), + (f'-rate(node_zfs_zpool_dataset_nwritten{{zpool=~"$pool"}}[$__rate_interval])', + "{{zpool}}/{{dataset}} write"), + ], + gp(0, 92, 12, 8), unit="Bps", + desc="Per-dataset throughput from node_exporter's objset kstats. Writes are drawn " + "negative.", +)) +panels.append(timeseries( + "Dataset IOPS", [ + (f'rate(node_zfs_zpool_dataset_reads{{zpool=~"$pool"}}[$__rate_interval])', + "{{zpool}}/{{dataset}} read"), + (f'-rate(node_zfs_zpool_dataset_writes{{zpool=~"$pool"}}[$__rate_interval])', + "{{zpool}}/{{dataset}} write"), + ], + gp(12, 92, 12, 8), unit="iops", + desc="Writes are drawn negative.", +)) + +panels.append(timeseries( + "ARC Size", [ + ("node_zfs_arc_size", "size"), + ("node_zfs_arc_c_max", "target max (c_max)"), + ("node_zfs_arc_c_min", "target min (c_min)"), + ], + gp(0, 100, 8, 8), unit="bytes", minval=0, +)) +panels.append(timeseries( + "ARC Hit Ratio", + [("rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min(" + "rate(node_zfs_arc_hits[$__rate_interval]) + " + "rate(node_zfs_arc_misses[$__rate_interval]), 1)", "hit ratio")], + gp(8, 100, 8, 8), unit="percentunit", minval=0, + desc="clamp_min avoids a divide-by-zero while the pool is idle.", +)) +panels.append(timeseries( + "ARC Hits / Misses", [ + ("rate(node_zfs_arc_hits[$__rate_interval])", "hits"), + ("rate(node_zfs_arc_misses[$__rate_interval])", "misses"), + ], + gp(16, 100, 8, 8), unit="ops", minval=0, +)) + +# ====================================================================================== +# Collapsed: advanced / diagnostic +# ====================================================================================== +panels.append(row("Pool Internals", 108, collapsed=True, children=[ + timeseries( + "Reclaim & Leaks", [ + (f'zfs_pool_freeing_bytes{{{POOL}}}', "{{pool}} freeing"), + (f'zfs_pool_leaked_bytes{{{POOL}}}', "{{pool}} leaked"), + (f'zfs_pool_checkpoint_bytes{{{POOL}}}', "{{pool}} checkpoint"), + ], + gp(0, 109, 8, 7), unit="bytes", minval=0, + desc="`freeing` is space still being reclaimed in the background after a " + "delete. `leaked` should always be zero.", + ), + timeseries( + "Dedup Ratio", [(f'zfs_pool_dedup_ratio{{{POOL}}}', "{{pool}}")], + gp(8, 109, 8, 7), minval=0, + desc="1.00 unless deduplication is enabled, which it usually should not be.", + ), + timeseries( + "Pool Health History", [ + (f'zfs_pool_health_code{{{POOL}}}', "{{pool}}"), + ], + # Bounded to the state-code range so the y-axis reads as states, not 0-100. + gp(16, 109, 8, 7), mappings=STATE_MAP, minval=0, maxval=8, + custom={"lineWidth": 2, "fillOpacity": 0, "lineInterpolation": "stepAfter"}, + desc="Health code over time — useful for spotting a pool that briefly " + "degraded and recovered. zfs_pool_scan_state is also collected if you " + "want to plot scrub/resilver phases alongside it.", + ), + table( + "Dataset Properties", [ + (f'zfs_dataset_recordsize_bytes{{{POOL}}}', None), + (f'zfs_dataset_quota_bytes{{{POOL}}}', None), + (f'zfs_dataset_refquota_bytes{{{POOL}}}', None), + (f'zfs_dataset_autosnapshot_enabled{{{POOL}}}', None), + ], + gp(0, 116, 24, 8), keep="dataset|Value.*", + renames={ + "__join__": "dataset", "dataset": "Dataset", + "Value #A": "Record Size", "Value #B": "Quota", + "Value #C": "Ref Quota", "Value #D": "Auto-snapshot", + }, + overrides=[ + {"matcher": {"id": "byName", "options": "Record Size"}, + "properties": [{"id": "unit", "value": "bytes"}]}, + {"matcher": {"id": "byName", "options": "Quota"}, + "properties": [{"id": "unit", "value": "bytes"}]}, + {"matcher": {"id": "byName", "options": "Ref Quota"}, + "properties": [{"id": "unit", "value": "bytes"}]}, + {"matcher": {"id": "byName", "options": "Auto-snapshot"}, + "properties": [{"id": "mappings", "value": [{"type": "value", "options": { + "0": {"text": "no", "color": "orange", "index": 0}, + "1": {"text": "yes", "color": "green", "index": 1}, + }}]}]}, + ], + desc="A quota of 0 means unlimited. Auto-snapshot reflects the " + "com.sun:auto-snapshot property that zfs-auto-snapshot honours.", + ), +])) + +panels.append(row("Collector Health", 109, collapsed=True, children=[ + timeseries( + "ZFS Collector Duration", + [("zfs_metrics_collection_duration_seconds", "collection")], + gp(0, 110, 8, 7), unit="s", minval=0, + desc="How long collectors/zfs-metrics.py takes. Listing every snapshot is the " + "slow part; if this approaches the 1 minute timer interval, raise " + "OnUnitActiveSec.", + ), + timeseries( + "Textfile Age", + [("time() - node_textfile_mtime_seconds", "{{file}}")], + gp(8, 110, 8, 7), unit="s", minval=0, + desc="Age of each .prom file. If this climbs without bound the collector has " + "stopped writing and every ZFS metric above is stale.", + ), + timeseries( + "Scrape Health", [ + ('up{job="node"}', "node exporter up"), + ("node_textfile_scrape_error", "textfile parse error"), + ], + gp(16, 110, 8, 7), minval=0, + desc="node_textfile_scrape_error goes to 1 if a .prom file is malformed, in " + "which case node_exporter drops it entirely.", + ), +])) + +dashboard = { + "title": "ZFS Pools", + "uid": "zfs-pools", + "description": "ZFS capacity, health, integrity, snapshot retention, dataset " + "properties and performance. Fed by node_exporter's zfs and systemd " + "collectors plus collectors/zfs-metrics.py from lgtm.nix.", + "tags": ["zfs", "storage"], + "timezone": "browser", + "schemaVersion": 41, + "version": 1, + "editable": True, + "graphTooltip": 1, + "refresh": "1m", + "time": {"from": "now-24h", "to": "now"}, + "templating": {"list": [{ + "name": "pool", + "label": "Pool", + "type": "query", + "datasource": DS, + "query": {"qryType": 1, "query": "label_values(zfs_pool_health_code, pool)", + "refId": "PrometheusVariableQueryEditor-VariableQuery"}, + "refresh": 1, + "includeAll": True, + "allValue": ".*", + "multi": True, + "current": {"text": ["All"], "value": ["$__all"]}, + "options": [], + "sort": 1, + }]}, + "annotations": {"list": [{ + "builtIn": 1, "datasource": {"type": "datasource", "uid": "grafana"}, + "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", "type": "dashboard", + }]}, + "panels": panels, +} + +print(json.dumps(dashboard, indent=2)) diff --git a/dashboards/panels.py b/dashboards/panels.py new file mode 100644 index 0000000..faa60f7 --- /dev/null +++ b/dashboards/panels.py @@ -0,0 +1,291 @@ +"""Shared Grafana panel builders for the dashboard generators in this directory. + +Hand-writing Grafana panel JSON at any volume invites subtle inconsistencies -- one +panel with a slightly different fieldConfig shape, a typo'd unit. Every panel here goes +through one code path per type, so they are structurally identical by construction. + +Used by gen_zfs.py and gen_smart.py. Both are run as scripts from this directory, so a +plain `from panels import ...` resolves. +""" + +DS = {"type": "prometheus", "uid": "prometheus"} + +# Thresholds: green until the value reaches 1, then red. For "any of these is a problem" +# counters. +OK_BAD = {"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "red", "value": 1}, +]} + +PLAIN = {"mode": "absolute", "steps": [{"color": "green", "value": None}]} + +_id = [0] + + +def nid(): + _id[0] += 1 + return _id[0] + + +def target(expr, legend=None, instant=False, fmt=None, ref="A"): + t = { + "datasource": DS, "editorMode": "code", "expr": expr, "refId": ref, + "instant": instant, "range": not instant, + } + if legend is not None: + t["legendFormat"] = legend + if fmt: + t["format"] = fmt + return t + + +def targets(specs, instant=False, fmt=None): + return [ + target(expr, legend, instant=instant, fmt=fmt, ref=chr(ord("A") + i)) + for i, (expr, legend) in enumerate(specs) + ] + + +def gp(x, y, w, h): + return {"h": h, "w": w, "x": x, "y": y} + + +def timeseries(title, specs, pos, unit=None, desc="", stack=False, minval=None, + maxval=None, custom=None, mappings=None): + cust = { + "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, + "fillOpacity": 40 if stack else 15, "showPoints": "never", + "spanNulls": True, "axisBorderShow": False, "axisCenteredZero": False, + "gradientMode": "none", "pointSize": 5, + "scaleDistribution": {"type": "linear"}, + "stacking": {"group": "A", "mode": "normal" if stack else "none"}, + "thresholdsStyle": {"mode": "off"}, + } + if custom: + cust.update(custom) + defaults = { + "color": {"mode": "palette-classic"}, + "custom": cust, + "mappings": mappings or [], + "thresholds": PLAIN, + } + if unit: + defaults["unit"] = unit + if minval is not None: + defaults["min"] = minval + if maxval is not None: + defaults["max"] = maxval + return { + "type": "timeseries", "title": title, "description": desc, + "id": nid(), "datasource": DS, "gridPos": pos, + "fieldConfig": {"defaults": defaults, "overrides": []}, + "options": { + "legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", + "placement": "bottom", "showLegend": True}, + "tooltip": {"mode": "multi", "sort": "desc"}, + }, + "targets": targets(specs), + } + + +def stat(title, specs, pos, unit=None, mappings=None, thresholds=None, desc="", + colormode="value", textmode="auto", graphmode="none", decimals=None, + no_value=None): + defaults = { + "color": {"mode": "thresholds"}, + "mappings": mappings or [], + "thresholds": thresholds or PLAIN, + } + if unit: + defaults["unit"] = unit + if decimals is not None: + defaults["decimals"] = decimals + if no_value is not None: + # Beats a bare "No data" when an empty result is itself the answer. + defaults["noValue"] = no_value + return { + "type": "stat", "title": title, "description": desc, + "id": nid(), "datasource": DS, "gridPos": pos, + "fieldConfig": {"defaults": defaults, "overrides": []}, + "options": { + "colorMode": colormode, "graphMode": graphmode, "justifyMode": "auto", + "orientation": "auto", "textMode": textmode, "wideLayout": True, + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + }, + "targets": targets(specs, instant=True), + } + + +def gauge(title, specs, pos, unit="percentunit", thresholds=None, desc="", maxv=1): + return { + "type": "gauge", "title": title, "description": desc, + "id": nid(), "datasource": DS, "gridPos": pos, + "fieldConfig": {"defaults": { + "color": {"mode": "thresholds"}, + "mappings": [], "min": 0, "max": maxv, "unit": unit, + "thresholds": thresholds or {"mode": "absolute", "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 0.8}, + {"color": "red", "value": 0.9}, + ]}, + }, "overrides": []}, + "options": { + "showThresholdLabels": False, "showThresholdMarkers": True, + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + }, + "targets": targets(specs, instant=True), + } + + +def bargauge(title, specs, pos, unit=None, thresholds=None, desc="", maxv=None, + display="basic"): + # "basic" draws one solid bar coloured by the value's own threshold. "lcd" splits + # the bar into segments coloured by position along the threshold scale, which makes + # a perfectly healthy value look mostly red — avoid it for anything with + # thresholds. + defaults = { + "color": {"mode": "thresholds"}, + "mappings": [], "min": 0, "thresholds": thresholds or PLAIN, + } + if unit: + defaults["unit"] = unit + if maxv is not None: + defaults["max"] = maxv + return { + "type": "bargauge", "title": title, "description": desc, + "id": nid(), "datasource": DS, "gridPos": pos, + "fieldConfig": {"defaults": defaults, "overrides": []}, + "options": { + "displayMode": display, "orientation": "horizontal", + "showUnfilled": True, "valueMode": "color", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + }, + "targets": targets(specs, instant=True), + } + + +def table(title, specs, pos, keep, renames, desc="", overrides=None, mappings=None, + sort_by=None, sort_desc=True, order=None, exclude=None): + """Instant-query table. + + `keep` is a regex of the columns to retain, applied before the join. Filtering + first is what stops duplicated label columns (Time, __name__, job, instance) from + colliding and being silently renamed to "field 1"/"field 2" by joinByField. + + When joining several queries, give exactly one of them the descriptive labels and + reduce the others with `max by (<join field>) (...)` so nothing collides. + """ + transformations = [ + {"id": "filterFieldsByName", "options": {"include": {"pattern": keep}}}, + ] + if len(specs) > 1: + join_field = renames.get("__join__", "dataset") + transformations.append( + {"id": "joinByField", "options": {"byField": join_field, "mode": "outer"}} + ) + transformations.append({"id": "organize", "options": { + # A synthetic join key built with label_join() has to survive `keep` so the + # join can use it, but should not be shown; drop it here. + "excludeByName": {name: True for name in (exclude or [])}, + # indexByName keys are the *original* field names, before renameByName runs. + "indexByName": {name: i for i, name in enumerate(order or [])}, + "renameByName": {k: v for k, v in renames.items() if k != "__join__"}, + }}) + if sort_by: + transformations.append( + {"id": "sortBy", "options": {"fields": {}, + "sort": [{"field": sort_by, + "desc": sort_desc}]}} + ) + return { + "type": "table", "title": title, "description": desc, + "id": nid(), "datasource": DS, "gridPos": pos, + "fieldConfig": {"defaults": { + "color": {"mode": "thresholds"}, + "mappings": mappings or [], + "custom": {"align": "auto", "cellOptions": {"type": "auto"}, + "inspect": False, "filterable": True}, + "thresholds": PLAIN, + }, "overrides": overrides or []}, + "options": {"showHeader": True, "cellHeight": "sm", + "footer": {"show": False, "reducer": ["sum"], "countRows": False, + "fields": ""}}, + "targets": targets(specs, instant=True, fmt="table"), + "transformations": transformations, + } + + +def row(title, y, collapsed=False, children=None, repeat=None): + """A dashboard row. + + Grafana keeps a *collapsed* row's children nested inside the row panel, and an + *expanded* row's children at the top level. Getting that backwards makes panels + vanish, so callers pass `children` only for collapsed rows. + + `repeat` names a template variable; Grafana then duplicates the row and everything + under it once per value of that variable. + """ + r = { + "type": "row", "title": title, "id": nid(), + "gridPos": gp(0, y, 24, 1), + "collapsed": collapsed, + "panels": children or [], + } + if repeat: + r["repeat"] = repeat + r["repeatDirection"] = "h" + return r + + +def query_var(name, query, label=None, multi=True, include_all=True, all_value=".*", + current_all=True, hide=0, regex=None): + """A Prometheus-backed template variable. + + `regex` filters and rewrites the returned strings. Named capture groups `value` and + `text` split one string into what queries substitute and what the UI shows -- nest + them (`(?<text>(?<value>\\S+) .*)`) to show a superset of what you select on. + """ + v = { + "name": name, + "label": label or name, + "type": "query", + "datasource": DS, + "query": {"qryType": 1, "query": query, + "refId": "PrometheusVariableQueryEditor-VariableQuery"}, + "refresh": 1, + "includeAll": include_all, + "multi": multi, + "options": [], + "sort": 1, + "hide": hide, + "regex": regex or "", + } + if include_all: + v["allValue"] = all_value + if current_all and include_all: + v["current"] = {"text": ["All"], "value": ["$__all"]} + return v + + +def dashboard(title, uid, description, panels, variables, tags, refresh="1m", + time_from="now-24h"): + return { + "title": title, + "uid": uid, + "description": description, + "tags": tags, + "timezone": "browser", + "schemaVersion": 41, + "version": 1, + "editable": True, + "graphTooltip": 1, + "refresh": refresh, + "time": {"from": time_from, "to": "now"}, + "templating": {"list": variables}, + "annotations": {"list": [{ + "builtIn": 1, "datasource": {"type": "datasource", "uid": "grafana"}, + "enable": True, "hide": True, "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", "type": "dashboard", + }]}, + "panels": panels, + } diff --git a/dashboards/smart.json b/dashboards/smart.json new file mode 100644 index 0000000..ed48cd1 --- /dev/null +++ b/dashboards/smart.json @@ -0,0 +1,3844 @@ +{ + "title": "Drive Health (SMART)", + "uid": "drive-health-smart", + "description": "Per-drive SMART health in the style of CrystalDiskInfo: a Good/Caution/Bad verdict, full raw attribute tables for SATA, SCSI error counters for SAS, and self-test history.", + "tags": [ + "smart", + "storage", + "hardware" + ], + "timezone": "browser", + "schemaVersion": 41, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "1m", + "time": { + "from": "now-24h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "ata_device", + "label": "SATA drive", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(smartctl:row{bus=\"ata\"}, row)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "includeAll": true, + "multi": true, + "options": [], + "sort": 1, + "hide": 0, + "regex": "/^(?<text>(?<value>\\S+)\\s.*)$/", + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + } + }, + { + "name": "sas_device", + "label": "SAS drive", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(smartctl:row{bus=\"sas\"}, row)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "includeAll": true, + "multi": true, + "options": [], + "sort": 1, + "hide": 0, + "regex": "/^(?<text>(?<value>\\S+)\\s.*)$/", + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + } + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Fleet Overview", + "id": 1, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "stat", + "title": "Healthy", + "description": "Drives with no failing attribute, no SCSI uncorrected errors and a passing SMART self-assessment.", + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(smartctl:health_code == 0) or vector(0)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "drives" + } + ] + }, + { + "type": "stat", + "title": "Caution", + "description": "Reallocated / pending / uncorrectable sectors on SATA, or a non-empty grown defect list on SAS. Not failing yet \u2014 worth watching the trend.", + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(smartctl:health_code == 1) or vector(0)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "drives" + } + ] + }, + { + "type": "stat", + "title": "Failing", + "description": "SMART self-assessment failed, an attribute is at or below its threshold, or the drive reported errors it could not correct. Replace these.", + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(smartctl:health_code == 2) or vector(0)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "drives" + } + ] + }, + { + "type": "stat", + "title": "Drives", + "description": "", + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(smartctl_device_smart_status)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "total" + } + ] + }, + { + "type": "stat", + "title": "Hottest Drive", + "description": "", + "id": 6, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 45 + }, + { + "color": "red", + "value": 55 + } + ] + }, + "unit": "celsius" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(smartctl_device_temperature{temperature_type=\"current\"})", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "max" + } + ] + }, + { + "type": "stat", + "title": "Longest Powered On", + "description": "", + "id": 7, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)))", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "max" + } + ] + }, + { + "type": "table", + "title": "All Drives", + "description": "Every drive, worst first. Health is computed by rules/smartctl.yml.", + "id": 8, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 10, + "w": 16, + "x": 0, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Health" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "GOOD", + "color": "green", + "index": 0 + }, + "1": { + "text": "CAUTION", + "color": "orange", + "index": 1 + }, + "2": { + "text": "BAD", + "color": "red", + "index": 2 + } + } + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "basic" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Temp" + }, + "properties": [ + { + "id": "unit", + "value": "celsius" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Powered On" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Capacity" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl:health_code * on(device) group_left(model_name, serial_number, protocol) smartctl_device", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (device) (smartctl_device_temperature{temperature_type=\"current\"})", + "refId": "B", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600))", + "refId": "C", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (device) (smartctl_device_capacity_bytes)", + "refId": "D", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "device|model_name|serial_number|protocol|Value.*" + } + } + }, + { + "id": "joinByField", + "options": { + "byField": "device", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "device": 0, + "model_name": 1, + "serial_number": 2, + "protocol": 3, + "Value #A": 4, + "Value #B": 5, + "Value #C": 6, + "Value #D": 7 + }, + "renameByName": { + "device": "Device", + "model_name": "Model", + "serial_number": "Serial", + "protocol": "Bus", + "Value #A": "Health", + "Value #B": "Temp", + "Value #C": "Powered On", + "Value #D": "Capacity" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Health", + "desc": true + } + ] + } + } + ] + }, + { + "type": "timeseries", + "title": "Temperatures", + "description": "Sustained operation above ~45C measurably shortens drive life.", + "id": 9, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "celsius" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_temperature{temperature_type=\"current\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{device}}" + } + ] + }, + { + "type": "row", + "title": "${ata_device:text}", + "id": 10, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "collapsed": false, + "panels": [], + "repeat": "ata_device", + "repeatDirection": "h" + }, + { + "type": "stat", + "title": "Health", + "description": "", + "id": 11, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "GOOD", + "color": "green", + "index": 0 + }, + "1": { + "text": "CAUTION", + "color": "orange", + "index": 1 + }, + "2": { + "text": "BAD", + "color": "red", + "index": 2 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl:health_code{device=~\"$ata_device\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "{{device}}" + } + ] + }, + { + "type": "stat", + "title": "Temperature", + "description": "", + "id": 12, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 45 + }, + { + "color": "red", + "value": 55 + } + ] + }, + "unit": "celsius" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_temperature{device=~\"$ata_device\", temperature_type=\"current\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "temp" + } + ] + }, + { + "type": "stat", + "title": "Powered On", + "description": "", + "id": 13, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$ata_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$ata_device\"} * 3600))", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "hours" + } + ] + }, + { + "type": "stat", + "title": "Capacity", + "description": "", + "id": 14, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_capacity_bytes{device=~\"$ata_device\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "bytes" + } + ] + }, + { + "type": "stat", + "title": "Power Cycles", + "description": "", + "id": 15, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_power_cycle_count{device=~\"$ata_device\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "cycles" + } + ] + }, + { + "type": "stat", + "title": "Reallocated Sectors", + "description": "Attribute 5 raw. Sectors the drive has already retired. Any growth here is the single most predictive sign of impending failure.", + "id": 16, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"5\", attribute_value_type=\"raw\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "sectors" + } + ] + }, + { + "type": "stat", + "title": "Pending Sectors", + "description": "Attribute 197 raw. Sectors that failed to read and are waiting to be reallocated on next write. Often precedes reallocations.", + "id": 17, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_id=\"197\", attribute_value_type=\"raw\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "sectors" + } + ] + }, + { + "type": "table", + "title": "Drive \u2014 $ata_device", + "description": "Identity of this drive. RPM shows SSD when the drive reports no rotation rate.", + "id": 18, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 20 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Capacity" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RPM" + }, + "properties": [ + { + "id": "unit", + "value": "rotrpm" + }, + { + "id": "noValue", + "value": "SSD" + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device{device=~\"$ata_device\"}", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$ata_device\"})", + "refId": "B", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$ata_device\"})", + "refId": "C", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "device|model_family|model_name|serial_number|firmware_version|interface|sata_version|form_factor|Value.*" + } + } + }, + { + "id": "joinByField", + "options": { + "byField": "device", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value #A": true + }, + "indexByName": { + "device": 0, + "model_family": 1, + "model_name": 2, + "serial_number": 3, + "firmware_version": 4, + "interface": 5, + "sata_version": 6, + "form_factor": 7, + "Value #B": 8, + "Value #C": 9 + }, + "renameByName": { + "device": "Device", + "Value #B": "RPM", + "Value #C": "Capacity", + "model_family": "Family", + "model_name": "Model", + "serial_number": "Serial", + "firmware_version": "Firmware", + "interface": "Interface", + "sata_version": "SATA", + "form_factor": "Form" + } + } + } + ] + }, + { + "type": "table", + "title": "SMART Attributes \u2014 $ata_device", + "description": "Every attribute the drive reports, with its raw value. Current/Worst are normalised 0-253 scores (higher is better); Threshold is where the vendor declares failure; Margin is Current minus Threshold. ID sorts as text, so the table is ordered by attribute name.", + "id": 19, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 24 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Margin" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "green", + "value": 25 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "basic" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"}", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"worst\"})", + "refId": "B", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})", + "refId": "C", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"raw\"})", + "refId": "D", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (attribute_id) (smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"value\"} - on(device, attribute_id) smartctl_device_attribute{device=~\"$ata_device\", attribute_value_type=\"thresh\"})", + "refId": "E", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "attribute_id|attribute_name|Value.*" + } + } + }, + { + "id": "joinByField", + "options": { + "byField": "attribute_id", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "attribute_id": 0, + "attribute_name": 1, + "Value #A": 2, + "Value #B": 3, + "Value #C": 4, + "Value #D": 5, + "Value #E": 6 + }, + "renameByName": { + "attribute_id": "ID", + "attribute_name": "Attribute", + "Value #A": "Current", + "Value #B": "Worst", + "Value #C": "Threshold", + "Value #D": "Raw", + "Value #E": "Margin" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Attribute", + "desc": false + } + ] + } + } + ] + }, + { + "type": "row", + "title": "${sas_device:text}", + "id": 20, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "collapsed": false, + "panels": [], + "repeat": "sas_device", + "repeatDirection": "h" + }, + { + "type": "stat", + "title": "Health", + "description": "", + "id": 21, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 37 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "GOOD", + "color": "green", + "index": 0 + }, + "1": { + "text": "CAUTION", + "color": "orange", + "index": 1 + }, + "2": { + "text": "BAD", + "color": "red", + "index": 2 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl:health_code{device=~\"$sas_device\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "{{device}}" + } + ] + }, + { + "type": "stat", + "title": "Temperature", + "description": "", + "id": 22, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 37 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 45 + }, + { + "color": "red", + "value": 55 + } + ] + }, + "unit": "celsius" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_temperature{device=~\"$sas_device\", temperature_type=\"current\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "temp" + } + ] + }, + { + "type": "stat", + "title": "Powered On", + "description": "SAS drives have no ATA attribute 9, so smartctl reports power_on_time from the Background Scan Results log page \u2014 a counter the background-scan subsystem maintains, not the drive's lifetime meter, and one that commonly reads far too low. This tile takes the larger of that counter and the lifetime-hours stamp in the drive's own self-test log, which is why it can exceed what `smartctl -x` prints as 'Accumulated power on time'.", + "id": 23, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 37 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (device) (smartctl_device_power_on_seconds{device=~\"$sas_device\"} or (smart_selftest_last_lifetime_hours{device=~\"$sas_device\"} * 3600))", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "hours" + } + ] + }, + { + "type": "stat", + "title": "Capacity", + "description": "", + "id": 24, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 37 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_capacity_bytes{device=~\"$sas_device\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "bytes" + } + ] + }, + { + "type": "stat", + "title": "Link Speed", + "description": "Negotiated SAS link rate. A drive that has fallen back to a lower rate is worth investigating \u2014 usually a cable or expander.", + "id": 25, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 37 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_interface_speed{device=~\"$sas_device\", speed_type=\"current\"} * 1000", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "bps" + } + ] + }, + { + "type": "stat", + "title": "Grown Defects", + "description": "Sectors reallocated since the drive left the factory \u2014 the SAS equivalent of ATA attribute 5. A stable non-zero count is tolerable; growth is not.", + "id": 26, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 37 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "defects" + } + ] + }, + { + "type": "stat", + "title": "Uncorrected Errors", + "description": "Errors the drive could not correct with ECC or retries, which means data was lost. Should be zero.", + "id": 27, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 37 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"} + smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "errors" + } + ] + }, + { + "type": "table", + "title": "Drive \u2014 $sas_device", + "description": "Identity of this drive. SAS drives leave firmware_version empty and report their revision in scsi_revision instead.", + "id": 28, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 41 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Capacity" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RPM" + }, + "properties": [ + { + "id": "unit", + "value": "rotrpm" + }, + { + "id": "noValue", + "value": "SSD" + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device{device=~\"$sas_device\"}", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (device) (smartctl_device_rotation_rate{device=~\"$sas_device\"})", + "refId": "B", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (device) (smartctl_device_capacity_bytes{device=~\"$sas_device\"})", + "refId": "C", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "device|scsi_vendor|model_name|serial_number|scsi_revision|scsi_version|protocol|form_factor|Value.*" + } + } + }, + { + "id": "joinByField", + "options": { + "byField": "device", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value #A": true + }, + "indexByName": { + "device": 0, + "scsi_vendor": 1, + "model_name": 2, + "serial_number": 3, + "scsi_revision": 4, + "scsi_version": 5, + "protocol": 6, + "form_factor": 7, + "Value #B": 8, + "Value #C": 9 + }, + "renameByName": { + "device": "Device", + "Value #B": "RPM", + "Value #C": "Capacity", + "scsi_vendor": "Vendor", + "model_name": "Model", + "serial_number": "Serial", + "scsi_revision": "Revision", + "scsi_version": "SCSI", + "protocol": "Bus", + "form_factor": "Form" + } + } + } + ] + }, + { + "type": "timeseries", + "title": "Corrected Errors \u2014 $sas_device", + "description": "Errors the drive fixed on its own. ECC-fast corrections are routine and grow constantly. Delayed corrections and especially rewrites/retries climbing is the signal worth watching.", + "id": 29, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_read_errors_corrected_by_eccfast{device=~\"$sas_device\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "read ECC fast" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_read_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}", + "refId": "B", + "instant": false, + "range": true, + "legendFormat": "read ECC delayed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_read_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}", + "refId": "C", + "instant": false, + "range": true, + "legendFormat": "read retries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_write_errors_corrected_by_eccfast{device=~\"$sas_device\"}", + "refId": "D", + "instant": false, + "range": true, + "legendFormat": "write ECC fast" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_write_errors_corrected_by_eccdelayed{device=~\"$sas_device\"}", + "refId": "E", + "instant": false, + "range": true, + "legendFormat": "write ECC delayed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_write_errors_corrected_by_rereads_rewrites{device=~\"$sas_device\"}", + "refId": "F", + "instant": false, + "range": true, + "legendFormat": "write retries" + } + ] + }, + { + "type": "timeseries", + "title": "Defects & Uncorrected \u2014 $sas_device", + "description": "Flat lines are good. Any upward step here is a real event.", + "id": 30, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "auto", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_scsi_grown_defect_list{device=~\"$sas_device\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "grown defects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_read_total_uncorrected_errors{device=~\"$sas_device\"}", + "refId": "B", + "instant": false, + "range": true, + "legendFormat": "read uncorrected" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_write_total_uncorrected_errors{device=~\"$sas_device\"}", + "refId": "C", + "instant": false, + "range": true, + "legendFormat": "write uncorrected" + } + ] + }, + { + "type": "row", + "title": "Self-Tests", + "id": 31, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 53 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "stat", + "title": "Drives Never Tested", + "description": "Drives with no self-test log entries at all. smartd only runs tests if its config includes an -s schedule; merely enabling the service does not.", + "id": 32, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 54 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(smartctl_device_smart_status) - count(count by (device) (smart_selftest_last_lifetime_hours)) or count(smartctl_device_smart_status)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "drives" + } + ] + }, + { + "type": "stat", + "title": "Failed Self-Tests", + "description": "", + "id": 33, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 54 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(smart_selftest_last_passed == 0) or vector(0)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "tests" + } + ] + }, + { + "type": "stat", + "title": "Oldest Long Test", + "description": "Powered-on hours since the least recently tested drive last completed an extended (long) self-test. Shows 'never run' when no drive has ever completed one \u2014 which is the default state, since smartd schedules no tests unless its config carries an -s directive.", + "id": 34, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 54 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 336 + }, + { + "color": "red", + "value": 1080 + } + ] + }, + "unit": "h", + "noValue": "never run" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{test_type=\"extended\"})", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "hours ago" + } + ] + }, + { + "type": "table", + "title": "Self-Test History", + "description": "Short and extended self-test results per drive, from collectors/smart-selftest.py. 'At Hours' is the drive's powered-on hour count when the test ran, straight from the log. 'Hours Ago' subtracts it from the drive's current powered-on hours; on SAS that figure already falls back to this log, so 0 means the newest log entry is also the highest hour count the drive has ever reported \u2014 not that the test ran just now.", + "id": 35, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 54 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Last Result" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "FAILED", + "color": "red", + "index": 0 + }, + "1": { + "text": "passed", + "color": "green", + "index": 1 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "basic" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_join(smart_selftest_last_passed, \"drive_test\", \"/\", \"device\", \"test_type\")", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (drive_test) (label_join(smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))", + "refId": "B", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (drive_test) (label_join(smart_selftest_count, \"drive_test\", \"/\", \"device\", \"test_type\"))", + "refId": "C", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (drive_test) (label_join(max by (device) (smartctl_device_power_on_seconds or (smart_selftest_last_lifetime_hours * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours, \"drive_test\", \"/\", \"device\", \"test_type\"))", + "refId": "D", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "drive_test|device|test_type|Value.*" + } + } + }, + { + "id": "joinByField", + "options": { + "byField": "drive_test", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "drive_test": true + }, + "indexByName": { + "device": 0, + "test_type": 1, + "Value #A": 2, + "Value #B": 3, + "Value #C": 4, + "Value #D": 5 + }, + "renameByName": { + "device": "Device", + "test_type": "Test", + "Value #A": "Last Result", + "Value #B": "At Hours", + "Value #C": "Logged", + "Value #D": "Hours Ago" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Device", + "desc": false + } + ] + } + } + ] + }, + { + "type": "row", + "title": "Attribute Trends", + "id": 41, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "collapsed": true, + "panels": [ + { + "type": "timeseries", + "title": "Reallocated Sectors (all SATA drives)", + "description": "", + "id": 36, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 65 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "auto", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_attribute{attribute_id=\"5\", attribute_value_type=\"raw\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{device}}" + } + ] + }, + { + "type": "timeseries", + "title": "Pending & Uncorrectable Sectors", + "description": "", + "id": 37, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 65 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "auto", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_attribute{attribute_id=~\"197|198\", attribute_value_type=\"raw\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{device}} {{attribute_name}}" + } + ] + }, + { + "type": "timeseries", + "title": "CRC Errors (cabling)", + "description": "Attribute 199 counts errors on the SATA link rather than the platters. Growth here usually means a cable or backplane, not a dying disk.", + "id": 38, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 65 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "auto", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_attribute{attribute_id=\"199\", attribute_value_type=\"raw\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{device}}" + } + ] + }, + { + "type": "timeseries", + "title": "Data Written / Read (SAS)", + "description": "", + "id": 39, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 72 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_bytes_written", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{device}} written" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_bytes_read", + "refId": "B", + "instant": false, + "range": true, + "legendFormat": "{{device}} read" + } + ] + }, + { + "type": "timeseries", + "title": "Health Code History", + "description": "", + "id": 40, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 72 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "GOOD", + "color": "green", + "index": 0 + }, + "1": { + "text": "CAUTION", + "color": "orange", + "index": 1 + }, + "2": { + "text": "BAD", + "color": "red", + "index": 2 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0, + "max": 2 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl:health_code", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{device}}" + } + ] + } + ] + }, + { + "type": "row", + "title": "Collector Health", + "id": 45, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "collapsed": true, + "panels": [ + { + "type": "timeseries", + "title": "smartctl Exit Status", + "description": "Non-zero means smartctl itself had trouble reading the device. Bit 0 is a command-line problem, bit 1 a device-open failure, bit 3 a failing drive.", + "id": 42, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 66 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_device_smartctl_exit_status", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{device}}" + } + ] + }, + { + "type": "timeseries", + "title": "Self-Test Collector Duration", + "description": "", + "id": 43, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 66 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smart_selftest_collection_duration_seconds", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "collection" + } + ] + }, + { + "type": "timeseries", + "title": "Devices Seen", + "description": "These should agree. A divergence means one of the two cannot see some drives.", + "id": 44, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 66 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smartctl_devices", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "exporter" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "smart_selftest_devices", + "refId": "B", + "instant": false, + "range": true, + "legendFormat": "self-test collector" + } + ] + } + ] + } + ] +} diff --git a/dashboards/zfs-pools.json b/dashboards/zfs-pools.json new file mode 100644 index 0000000..0721bb8 --- /dev/null +++ b/dashboards/zfs-pools.json @@ -0,0 +1,4372 @@ +{ + "title": "ZFS Pools", + "uid": "zfs-pools", + "description": "ZFS capacity, health, integrity, snapshot retention, dataset properties and performance. Fed by node_exporter's zfs and systemd collectors plus collectors/zfs-metrics.py from lgtm.nix.", + "tags": [ + "zfs", + "storage" + ], + "timezone": "browser", + "schemaVersion": 41, + "version": 1, + "editable": true, + "graphTooltip": 1, + "refresh": "1m", + "time": { + "from": "now-24h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "pool", + "label": "Pool", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "qryType": 1, + "query": "label_values(zfs_pool_health_code, pool)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "options": [], + "sort": 1 + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Pool Overview", + "id": 1, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "stat", + "title": "Pool Health", + "description": "Health as reported by `zpool list -o health`.", + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "ONLINE", + "color": "green", + "index": 0 + }, + "1": { + "text": "DEGRADED", + "color": "orange", + "index": 1 + }, + "2": { + "text": "FAULTED", + "color": "red", + "index": 2 + }, + "3": { + "text": "OFFLINE", + "color": "text", + "index": 3 + }, + "4": { + "text": "UNAVAIL", + "color": "red", + "index": 4 + }, + "5": { + "text": "REMOVED", + "color": "purple", + "index": 5 + }, + "6": { + "text": "SUSPENDED", + "color": "red", + "index": 6 + }, + "7": { + "text": "AVAIL", + "color": "blue", + "index": 7 + }, + "8": { + "text": "UNKNOWN", + "color": "text", + "index": 8 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_health_code{pool=~\"$pool\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "{{pool}}" + } + ] + }, + { + "type": "stat", + "title": "Data Errors", + "description": "Permanent, unrepairable errors from `zpool status`. Anything above zero means data has been lost and the affected files need restoring.", + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(zfs_pool_data_errors{pool=~\"$pool\"}) or vector(0)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "errors" + } + ] + }, + { + "type": "stat", + "title": "vdev Errors", + "description": "Total read + write + checksum errors across all vdevs. Cleared by `zpool clear`.", + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(zfs_pool_vdev_errors_total{pool=~\"$pool\"}) or vector(0)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "errors" + } + ] + }, + { + "type": "stat", + "title": "Last Scrub", + "description": "Time since the most recent completed scrub. Turns amber past ~5 weeks, which is a monthly scrub schedule having missed a run.", + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3024000 + }, + { + "color": "red", + "value": 6048000 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - max(zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"})", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "ago" + } + ] + }, + { + "type": "stat", + "title": "Scrub / Resilver", + "description": "", + "id": 6, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "idle", + "color": "green", + "index": 0 + }, + "1": { + "text": "RUNNING", + "color": "blue", + "index": 1 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(zfs_pool_scrub_in_progress{pool=~\"$pool\"})", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "scrub" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(zfs_pool_resilver_in_progress{pool=~\"$pool\"})", + "refId": "B", + "instant": true, + "range": false, + "legendFormat": "resilver" + } + ] + }, + { + "type": "stat", + "title": "Locked Datasets", + "description": "Encrypted datasets whose key is not loaded. These are unreadable and are silently skipped by anything that walks the filesystem, including backups.", + "id": 7, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(zfs_dataset_encrypted{pool=~\"$pool\"} == 1 and zfs_dataset_key_available{pool=~\"$pool\"} == 0) or vector(0)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "locked" + } + ] + }, + { + "type": "gauge", + "title": "Space Used", + "description": "How full the pool is: allocated / total. ZFS write performance degrades sharply past ~80%, and the pool becomes effectively read-only near 100%. The byte figures behind this are in 'Capacity by Pool' below.", + "id": 8, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "max": 1, + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + }, + "overrides": [] + }, + "options": { + "showThresholdLabels": false, + "showThresholdMarkers": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_capacity_ratio{pool=~\"$pool\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "{{pool}}" + } + ] + }, + { + "type": "gauge", + "title": "Fragmentation", + "description": "Free-space fragmentation.", + "id": 9, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 5, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "max": 1, + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 0.75 + } + ] + } + }, + "overrides": [] + }, + "options": { + "showThresholdLabels": false, + "showThresholdMarkers": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "{{pool}}" + } + ] + }, + { + "type": "stat", + "title": "Free Space", + "description": "", + "id": 10, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 10, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "{{pool}}" + } + ] + }, + { + "type": "stat", + "title": "Snapshots Overdue", + "description": "Dataset/label pairs whose newest snapshot is more than two intervals old \u2014 i.e. the auto-snapshot timer has missed at least one run.", + "id": 11, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 15, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count((time() - zfs_snapshot_newest_timestamp_seconds{pool=~\"$pool\"}) > on(label) group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "overdue" + } + ] + }, + { + "type": "stat", + "title": "Retention Depth", + "description": "Age of the oldest snapshot on the pool \u2014 how far back you can actually restore from.", + "id": 12, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "oldest" + } + ] + }, + { + "type": "timeseries", + "title": "Pool Space Used / Free", + "description": "Stacked, so the top of each pool's band is its total size and the split between the bands is the usage fraction the gauge above shows.", + "id": 13, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 40, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_allocated_bytes{pool=~\"$pool\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{pool}} used" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", + "refId": "B", + "instant": false, + "range": true, + "legendFormat": "{{pool}} free" + } + ] + }, + { + "type": "timeseries", + "title": "Fragmentation", + "description": "Free-space fragmentation \u2014 how scattered the pool's remaining free space is, not how scattered the data is. It rises as a pool fills and stays high afterwards; only rewriting data (send/recv, or deleting a lot) brings it down. Kept on its own axis because it is unrelated to space used, and plotting the two together invites reading one as the other.", + "id": 14, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{pool}}" + } + ] + }, + { + "type": "table", + "title": "Capacity by Pool", + "description": "Per-pool space in bytes. 'Size' is the raw pool size zpool reports, which for raidz is before parity overhead \u2014 so 'Free' is the number to trust for how much you can still write. 'Snapshots' is space held only by snapshots.", + "id": 15, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 17 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^(Size|Used|Free|Snapshots)$" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "gradient" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Fragmentation" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (pool) (zfs_pool_allocated_bytes{pool=~\"$pool\"})", + "refId": "B", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (pool) (zfs_pool_free_bytes{pool=~\"$pool\"})", + "refId": "C", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (pool) (zfs_pool_capacity_ratio{pool=~\"$pool\"})", + "refId": "D", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (pool) (zfs_pool_fragmentation_ratio{pool=~\"$pool\"})", + "refId": "E", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (pool) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})", + "refId": "F", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (pool) (zfs_pool_dedup_ratio{pool=~\"$pool\"})", + "refId": "G", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "pool|Value.*" + } + } + }, + { + "id": "joinByField", + "options": { + "byField": "pool", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "pool": 0, + "Value #A": 1, + "Value #B": 2, + "Value #C": 3, + "Value #D": 4, + "Value #E": 5, + "Value #F": 6, + "Value #G": 7 + }, + "renameByName": { + "pool": "Pool", + "Value #A": "Size", + "Value #B": "Used", + "Value #C": "Free", + "Value #D": "Used %", + "Value #E": "Fragmentation", + "Value #F": "Snapshots", + "Value #G": "Dedup" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Used %", + "desc": true + } + ] + } + } + ] + }, + { + "type": "row", + "title": "Integrity & Errors", + "id": 16, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "table", + "title": "vdev State", + "description": "Every vdev and leaf device in the selected pool(s).", + "id": 17, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "ONLINE", + "color": "green", + "index": 0 + }, + "1": { + "text": "DEGRADED", + "color": "orange", + "index": 1 + }, + "2": { + "text": "FAULTED", + "color": "red", + "index": 2 + }, + "3": { + "text": "OFFLINE", + "color": "text", + "index": 3 + }, + "4": { + "text": "UNAVAIL", + "color": "red", + "index": 4 + }, + "5": { + "text": "REMOVED", + "color": "purple", + "index": 5 + }, + "6": { + "text": "SUSPENDED", + "color": "red", + "index": 6 + }, + "7": { + "text": "AVAIL", + "color": "blue", + "index": 7 + }, + "8": { + "text": "UNKNOWN", + "color": "text", + "index": 8 + } + } + } + ], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "State" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "basic" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_vdev_state_code{pool=~\"$pool\"}", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "pool|vdev|Value" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "pool": "Pool", + "vdev": "vdev", + "Value": "State" + } + } + } + ] + }, + { + "type": "timeseries", + "title": "vdev Error Counters", + "description": "Cumulative per-vdev errors. A counter that climbs is a disk on its way out.", + "id": 18, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "auto", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_vdev_errors_total{pool=~\"$pool\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{pool}}/{{vdev}} {{type}}" + } + ] + }, + { + "type": "table", + "title": "Scrub & Resilver History", + "description": "Result of the last completed scrub per pool. Errors here are blocks that could not be repaired from redundancy.", + "id": 19, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 35 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Scrub Finished" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Resilver Finished" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Repaired" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Errors" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "basic" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"} * 1000", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_last_scrub_duration_seconds{pool=~\"$pool\"}", + "refId": "B", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_last_scrub_errors{pool=~\"$pool\"}", + "refId": "C", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_last_scrub_repaired_bytes{pool=~\"$pool\"}", + "refId": "D", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_last_resilver_end_timestamp_seconds{pool=~\"$pool\"} * 1000", + "refId": "E", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "pool|Value.*" + } + } + }, + { + "id": "joinByField", + "options": { + "byField": "pool", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "pool": "Pool", + "Value #A": "Scrub Finished", + "Value #B": "Duration", + "Value #C": "Errors", + "Value #D": "Repaired", + "Value #E": "Resilver Finished" + } + } + } + ] + }, + { + "type": "timeseries", + "title": "Pool Data Errors", + "description": "Permanent data errors over time.", + "id": 20, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 35 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "auto", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_data_errors{pool=~\"$pool\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{pool}}" + } + ] + }, + { + "type": "row", + "title": "Snapshots & Retention", + "id": 21, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "bargauge", + "title": "Time Since Last Snapshot", + "description": "Age of the newest snapshot of each label. Compare against the label's own cadence \u2014 'frequent' is every 15 minutes, 'monthly' every month, so the colour thresholds here are only a coarse hint.", + "id": 22, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 43 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 7200 + }, + { + "color": "red", + "value": 172800 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "color", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{pool=~\"$pool\"})", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "{{label}}" + } + ] + }, + { + "type": "bargauge", + "title": "Snapshots vs Retention Target", + "description": "Worst dataset's snapshot count as a fraction of the configured services.zfs.autoSnapshot retention. 100% means the full set is on disk; a freshly-created dataset legitimately sits below target until it fills up.", + "id": 23, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 43 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "unit": "percentunit", + "max": 1.2 + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "color", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min by (label) (zfs_snapshot_count{pool=~\"$pool\"} / on(label) group_left() zfs_snapshot_retention_target)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "{{label}}" + } + ] + }, + { + "type": "bargauge", + "title": "Oldest Snapshot", + "description": "How far back each label reaches.", + "id": 24, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 43 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "color", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - min by (label) (zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "{{label}}" + } + ] + }, + { + "type": "timeseries", + "title": "Snapshot Count by Label", + "description": "Total snapshots per label. A steady sawtooth is healthy rotation; a monotonic decline means snapshots are expiring but not being replaced.", + "id": 25, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (label) (zfs_snapshot_count{pool=~\"$pool\"})", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{label}}" + } + ] + }, + { + "type": "timeseries", + "title": "Snapshot Space by Label", + "description": "Space held down by snapshots \u2014 the cost of your retention policy.", + "id": 26, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 40, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (label) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{label}}" + } + ] + }, + { + "type": "stat", + "title": "Snapshot / Scrub Job Failures", + "description": "systemd units for the auto-snapshot and auto-scrub timers currently in a failed state.", + "id": 27, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 59 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "wideLayout": true, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(node_systemd_unit_state{name=~\"zfs-(snapshot|scrub).*\",state=\"failed\"}) or vector(0)", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "failed units" + } + ] + }, + { + "type": "bargauge", + "title": "Time Since Timer Last Fired", + "description": "When each zfs-* systemd timer last triggered. This is the job side of the picture; the snapshot ages above are the result side.", + "id": 28, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 59 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "color", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - node_systemd_timer_last_trigger_seconds{name=~\"zfs-.*\"}", + "refId": "A", + "instant": true, + "range": false, + "legendFormat": "{{name}}" + } + ] + }, + { + "type": "row", + "title": "Datasets", + "id": 29, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "title": "Dataset Usage Over Time", + "description": "Space each dataset is responsible for, including its snapshots and descendants \u2014 so a parent's line contains its children's. Unstacked and unfilled so a single dataset's growth is readable against the others rather than riding on top of them. The legend carries last and max per dataset; the instant breakdown is in 'Capacity by Dataset' below.", + "id": 30, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 66 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "auto", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{dataset}}" + } + ] + }, + { + "type": "table", + "title": "Capacity by Dataset", + "description": "Per-dataset space, separate from the per-pool figures above. 'Used' is the total a dataset is responsible for; 'Own Data' and 'Snapshots' are the two halves of that. 'Logical' is the size before compression, so Logical / Referenced is where the compression ratio comes from. 'Available' is shared with every sibling dataset unless a quota says otherwise.", + "id": 31, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 75 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^(Used|Own Data|Snapshots|Referenced|Logical|Available|Quota)$" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Quota" + }, + "properties": [ + { + "id": "noValue", + "value": "none" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Ratio" + }, + "properties": [ + { + "id": "decimals", + "value": 2 + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (dataset) (zfs_dataset_used_by_dataset_bytes{pool=~\"$pool\"})", + "refId": "B", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (dataset) (zfs_dataset_used_by_snapshots_bytes{pool=~\"$pool\"})", + "refId": "C", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (dataset) (zfs_dataset_referenced_bytes{pool=~\"$pool\"})", + "refId": "D", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (dataset) (zfs_dataset_logical_used_bytes{pool=~\"$pool\"})", + "refId": "E", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (dataset) (zfs_dataset_available_bytes{pool=~\"$pool\"})", + "refId": "F", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (dataset) (zfs_dataset_compression_ratio{pool=~\"$pool\"})", + "refId": "G", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (dataset) (zfs_dataset_quota_bytes{pool=~\"$pool\"} > 0)", + "refId": "H", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "pool|dataset|Value.*" + } + } + }, + { + "id": "joinByField", + "options": { + "byField": "dataset", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "pool": 0, + "dataset": 1, + "Value #A": 2, + "Value #B": 3, + "Value #C": 4, + "Value #D": 5, + "Value #E": 6, + "Value #F": 7, + "Value #G": 8, + "Value #H": 9 + }, + "renameByName": { + "dataset": "Dataset", + "pool": "Pool", + "Value #A": "Used", + "Value #B": "Own Data", + "Value #C": "Snapshots", + "Value #D": "Referenced", + "Value #E": "Logical", + "Value #F": "Available", + "Value #G": "Ratio", + "Value #H": "Quota" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Used", + "desc": true + } + ] + } + } + ] + }, + { + "type": "table", + "title": "Compression", + "description": "group_left() attaches the algorithm label from the info metric onto the ratio, so this needs only one query and no join transformation.", + "id": 32, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_dataset_compression_ratio{pool=~\"$pool\"} * on(pool, dataset) group_left(compression) zfs_dataset_compression{pool=~\"$pool\"}", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "pool|dataset|compression|Value" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "pool": 0, + "dataset": 1, + "compression": 2, + "Value": 3 + }, + "renameByName": { + "pool": "Pool", + "dataset": "Dataset", + "compression": "Algorithm", + "Value": "Ratio" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Ratio", + "desc": true + } + ] + } + } + ] + }, + { + "type": "table", + "title": "Encryption", + "description": "Encryption suite per dataset and the state of its key. Datasets that are not encrypted show suite 'off' and a dash \u2014 only an encrypted dataset can be locked.", + "id": 33, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "\u2014", + "color": "text", + "index": 0 + }, + "1": { + "text": "unlocked", + "color": "green", + "index": 1 + }, + "2": { + "text": "LOCKED", + "color": "red", + "index": 2 + }, + "3": { + "text": "unknown", + "color": "orange", + "index": 3 + } + } + } + ], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Key" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-background", + "mode": "basic" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_dataset_keystatus_code{pool=~\"$pool\"} * on(pool, dataset) group_left(encryption) zfs_dataset_encryption{pool=~\"$pool\"}", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "pool|dataset|encryption|Value" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "pool": 0, + "dataset": 1, + "encryption": 2, + "Value": 3 + }, + "renameByName": { + "pool": "Pool", + "dataset": "Dataset", + "encryption": "Suite", + "Value": "Key" + } + } + } + ] + }, + { + "type": "row", + "title": "Performance", + "id": 34, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 91 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "title": "Dataset Throughput", + "description": "Per-dataset throughput from node_exporter's objset kstats. Writes are drawn negative.", + "id": 35, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 92 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_zfs_zpool_dataset_nread{zpool=~\"$pool\"}[$__rate_interval])", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{zpool}}/{{dataset}} read" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "-rate(node_zfs_zpool_dataset_nwritten{zpool=~\"$pool\"}[$__rate_interval])", + "refId": "B", + "instant": false, + "range": true, + "legendFormat": "{{zpool}}/{{dataset}} write" + } + ] + }, + { + "type": "timeseries", + "title": "Dataset IOPS", + "description": "Writes are drawn negative.", + "id": 36, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 92 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_zfs_zpool_dataset_reads{zpool=~\"$pool\"}[$__rate_interval])", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{zpool}}/{{dataset}} read" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "-rate(node_zfs_zpool_dataset_writes{zpool=~\"$pool\"}[$__rate_interval])", + "refId": "B", + "instant": false, + "range": true, + "legendFormat": "{{zpool}}/{{dataset}} write" + } + ] + }, + { + "type": "timeseries", + "title": "ARC Size", + "description": "", + "id": 37, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 100 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_zfs_arc_size", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "size" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_zfs_arc_c_max", + "refId": "B", + "instant": false, + "range": true, + "legendFormat": "target max (c_max)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_zfs_arc_c_min", + "refId": "C", + "instant": false, + "range": true, + "legendFormat": "target min (c_min)" + } + ] + }, + { + "type": "timeseries", + "title": "ARC Hit Ratio", + "description": "clamp_min avoids a divide-by-zero while the pool is idle.", + "id": 38, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 100 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min(rate(node_zfs_arc_hits[$__rate_interval]) + rate(node_zfs_arc_misses[$__rate_interval]), 1)", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "hit ratio" + } + ] + }, + { + "type": "timeseries", + "title": "ARC Hits / Misses", + "description": "", + "id": 39, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 100 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_zfs_arc_hits[$__rate_interval])", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "hits" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(node_zfs_arc_misses[$__rate_interval])", + "refId": "B", + "instant": false, + "range": true, + "legendFormat": "misses" + } + ] + }, + { + "type": "row", + "title": "Pool Internals", + "id": 44, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 108 + }, + "collapsed": true, + "panels": [ + { + "type": "timeseries", + "title": "Reclaim & Leaks", + "description": "`freeing` is space still being reclaimed in the background after a delete. `leaked` should always be zero.", + "id": 40, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 109 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_freeing_bytes{pool=~\"$pool\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{pool}} freeing" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_leaked_bytes{pool=~\"$pool\"}", + "refId": "B", + "instant": false, + "range": true, + "legendFormat": "{{pool}} leaked" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_checkpoint_bytes{pool=~\"$pool\"}", + "refId": "C", + "instant": false, + "range": true, + "legendFormat": "{{pool}} checkpoint" + } + ] + }, + { + "type": "timeseries", + "title": "Dedup Ratio", + "description": "1.00 unless deduplication is enabled, which it usually should not be.", + "id": 41, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 109 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_dedup_ratio{pool=~\"$pool\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{pool}}" + } + ] + }, + { + "type": "timeseries", + "title": "Pool Health History", + "description": "Health code over time \u2014 useful for spotting a pool that briefly degraded and recovered. zfs_pool_scan_state is also collected if you want to plot scrub/resilver phases alongside it.", + "id": 42, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 109 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "ONLINE", + "color": "green", + "index": 0 + }, + "1": { + "text": "DEGRADED", + "color": "orange", + "index": 1 + }, + "2": { + "text": "FAULTED", + "color": "red", + "index": 2 + }, + "3": { + "text": "OFFLINE", + "color": "text", + "index": 3 + }, + "4": { + "text": "UNAVAIL", + "color": "red", + "index": 4 + }, + "5": { + "text": "REMOVED", + "color": "purple", + "index": 5 + }, + "6": { + "text": "SUSPENDED", + "color": "red", + "index": 6 + }, + "7": { + "text": "AVAIL", + "color": "blue", + "index": 7 + }, + "8": { + "text": "UNKNOWN", + "color": "text", + "index": 8 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0, + "max": 8 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_pool_health_code{pool=~\"$pool\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{pool}}" + } + ] + }, + { + "type": "table", + "title": "Dataset Properties", + "description": "A quota of 0 means unlimited. Auto-snapshot reflects the com.sun:auto-snapshot property that zfs-auto-snapshot honours.", + "id": 43, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 116 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Record Size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Quota" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Ref Quota" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Auto-snapshot" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "no", + "color": "orange", + "index": 0 + }, + "1": { + "text": "yes", + "color": "green", + "index": 1 + } + } + } + ] + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_dataset_recordsize_bytes{pool=~\"$pool\"}", + "refId": "A", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_dataset_quota_bytes{pool=~\"$pool\"}", + "refId": "B", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_dataset_refquota_bytes{pool=~\"$pool\"}", + "refId": "C", + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_dataset_autosnapshot_enabled{pool=~\"$pool\"}", + "refId": "D", + "instant": true, + "range": false, + "format": "table" + } + ], + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "dataset|Value.*" + } + } + }, + { + "id": "joinByField", + "options": { + "byField": "dataset", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "dataset": "Dataset", + "Value #A": "Record Size", + "Value #B": "Quota", + "Value #C": "Ref Quota", + "Value #D": "Auto-snapshot" + } + } + } + ] + } + ] + }, + { + "type": "row", + "title": "Collector Health", + "id": 48, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 109 + }, + "collapsed": true, + "panels": [ + { + "type": "timeseries", + "title": "ZFS Collector Duration", + "description": "How long collectors/zfs-metrics.py takes. Listing every snapshot is the slow part; if this approaches the 1 minute timer interval, raise OnUnitActiveSec.", + "id": 45, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 110 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "zfs_metrics_collection_duration_seconds", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "collection" + } + ] + }, + { + "type": "timeseries", + "title": "Textfile Age", + "description": "Age of each .prom file. If this climbs without bound the collector has stopped writing and every ZFS metric above is stale.", + "id": 46, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 110 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - node_textfile_mtime_seconds", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "{{file}}" + } + ] + }, + { + "type": "timeseries", + "title": "Scrape Health", + "description": "node_textfile_scrape_error goes to 1 if a .prom file is malformed, in which case node_exporter drops it entirely.", + "id": 47, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 110 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 15, + "showPoints": "never", + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "gradientMode": "none", + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "up{job=\"node\"}", + "refId": "A", + "instant": false, + "range": true, + "legendFormat": "node exporter up" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_textfile_scrape_error", + "refId": "B", + "instant": false, + "range": true, + "legendFormat": "textfile parse error" + } + ] + } + ] + } + ] +} |