diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | README.md | 17 | ||||
| -rw-r--r-- | dashboards/smart.json | 3865 | ||||
| -rw-r--r-- | dashboards/zfs-pools.json | 4395 | ||||
| -rw-r--r-- | flake.nix | 125 | ||||
| -rw-r--r-- | modules/dashboards.nix | 52 |
6 files changed, 119 insertions, 8338 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b49e032 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# Generated by dashboards/gen_*.py; the build runs the generators itself. +dashboards/*.json +__pycache__/ @@ -17,8 +17,7 @@ What *has* been checked: - every dashboard query executed against a real Prometheus fed fixture data - `promtool` over the recording rules and the generated scrape config -- `nix flake check`: both modules evaluate, and the committed dashboard JSON matches what - the generators produce +- `nix flake check`: both modules evaluate, and both dashboard generators run - the collectors run against mock `smartctl`/`zpool`/`zfs`, and partially against real hardware @@ -225,21 +224,23 @@ modules/ dashboards.nix builds dashboard JSON into a provisionable form packages.nix the two collectors as ordinary packages collectors/*.py the collectors themselves; runnable by hand -dashboards/gen_*.py dashboard generators -dashboards/*.json generated, committed, checked +dashboards/gen_*.py dashboard generators; the JSON is built, not committed rules/smartctl.yml Prometheus recording rules ``` ## Working on it -Dashboard JSON is **generated** — edit `dashboards/gen_*.py`, never the JSON: +The ZFS Pools and Drive Health dashboards are **generated at build time** from +`dashboards/gen_*.py`; no JSON is committed, so there is nothing to keep in sync and no +way to ship a stale dashboard. To see what a generator currently produces: ```sh -python3 dashboards/gen_zfs.py > dashboards/zfs-pools.json -python3 dashboards/gen_smart.py > dashboards/smart.json +python3 dashboards/gen_zfs.py | less +nix build .#nixosConfigurations.<host>.config.system.build.toplevel # or just deploy ``` -`nix flake check` fails if you forget. +(Flakes copy only git-tracked files into the store, so "gitignore the JSON and read it +from disk anyway" is not an option — it is generate, or commit.) The collectors are plain scripts taking `--output PATH`, so you can see exactly what a machine reports: diff --git a/dashboards/smart.json b/dashboards/smart.json deleted file mode 100644 index f6c9867..0000000 --- a/dashboards/smart.json +++ /dev/null @@ -1,3865 +0,0 @@ -{ - "title": "Drive Health (SMART)", - "uid": "drive-health-smart", - "description": "Per-drive SMART health in the style of CrystalDiskInfo: a Good/Caution/Bad verdict, full raw attribute tables for SATA, SCSI error counters for SAS, and self-test history.", - "tags": [ - "smart", - "storage", - "hardware" - ], - "timezone": "browser", - "schemaVersion": 41, - "version": 1, - "editable": true, - "graphTooltip": 1, - "refresh": "1m", - "time": { - "from": "now-24h", - "to": "now" - }, - "templating": { - "list": [ - { - "name": "host", - "label": "Host", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "query": { - "qryType": 1, - "query": "label_values(smartctl_device_smart_status, host)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "includeAll": false, - "multi": false, - "options": [], - "sort": 1, - "hide": 0, - "regex": "" - }, - { - "name": "ata_device", - "label": "SATA drive", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "query": { - "qryType": 1, - "query": "label_values(smartctl:row{bus=\"ata\", host=~\"$host\"}, row)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "includeAll": true, - "multi": true, - "options": [], - "sort": 1, - "hide": 0, - "regex": "/^(?<text>(?<value>\\S+)\\s.*)$/", - "allValue": ".*", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - } - }, - { - "name": "sas_device", - "label": "SAS drive", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "query": { - "qryType": 1, - "query": "label_values(smartctl:row{bus=\"sas\", host=~\"$host\"}, row)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "includeAll": true, - "multi": true, - "options": [], - "sort": 1, - "hide": 0, - "regex": "/^(?<text>(?<value>\\S+)\\s.*)$/", - "allValue": ".*", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - } - } - ] - }, - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "panels": [ - { - "type": "row", - "title": "Fleet Overview", - "id": 1, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "collapsed": false, - "panels": [] - }, - { - "type": "stat", - "title": "Healthy", - "description": "Drives with no failing attribute, no SCSI uncorrected errors and a passing SMART self-assessment.", - "id": 2, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count(smartctl:health_code{host=~\"$host\"} == 0) or vector(0)", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "drives" - } - ] - }, - { - "type": "stat", - "title": "Caution", - "description": "Reallocated / pending / uncorrectable sectors on SATA, or a non-empty grown defect list on SAS. Not failing yet \u2014 worth watching the trend.", - "id": 3, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count(smartctl:health_code{host=~\"$host\"} == 1) or vector(0)", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "drives" - } - ] - }, - { - "type": "stat", - "title": "Failing", - "description": "SMART self-assessment failed, an attribute is at or below its threshold, or the drive reported errors it could not correct. Replace these.", - "id": 4, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count(smartctl:health_code{host=~\"$host\"} == 2) or vector(0)", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "drives" - } - ] - }, - { - "type": "stat", - "title": "Drives", - "description": "", - "id": 5, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count(smartctl_device_smart_status{host=~\"$host\"})", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "total" - } - ] - }, - { - "type": "stat", - "title": "Hottest Drive", - "description": "", - "id": 6, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 45 - }, - { - "color": "red", - "value": 55 - } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"})", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "max" - } - ] - }, - { - "type": "stat", - "title": "Longest Powered On", - "description": "", - "id": 7, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)))", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "max" - } - ] - }, - { - "type": "table", - "title": "All Drives", - "description": "Every drive, worst first. Health is computed by rules/smartctl.yml.", - "id": 8, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 10, - "w": 16, - "x": 0, - "y": 5 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Health" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "type": "value", - "options": { - "0": { - "text": "GOOD", - "color": "green", - "index": 0 - }, - "1": { - "text": "CAUTION", - "color": "orange", - "index": 1 - }, - "2": { - "text": "BAD", - "color": "red", - "index": 2 - } - } - } - ] - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - }, - { - "color": "red", - "value": 2 - } - ] - } - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background", - "mode": "basic" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Temp" - }, - "properties": [ - { - "id": "unit", - "value": "celsius" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Powered On" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Capacity" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl:health_code{host=~\"$host\"} * on(device) group_left(model_name, serial_number, protocol) smartctl_device{host=~\"$host\"}", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (device) (smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"})", - "refId": "B", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600))", - "refId": "C", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\"})", - "refId": "D", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "device|model_name|serial_number|protocol|Value.*" - } - } - }, - { - "id": "joinByField", - "options": { - "byField": "device", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "device": 0, - "model_name": 1, - "serial_number": 2, - "protocol": 3, - "Value #A": 4, - "Value #B": 5, - "Value #C": 6, - "Value #D": 7 - }, - "renameByName": { - "device": "Device", - "model_name": "Model", - "serial_number": "Serial", - "protocol": "Bus", - "Value #A": "Health", - "Value #B": "Temp", - "Value #C": "Powered On", - "Value #D": "Capacity" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "Health", - "desc": true - } - ] - } - } - ] - }, - { - "type": "timeseries", - "title": "Temperatures", - "description": "Sustained operation above ~45C measurably shortens drive life.", - "id": 9, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 16, - "y": 5 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_temperature{host=~\"$host\", temperature_type=\"current\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{device}}" - } - ] - }, - { - "type": "row", - "title": "${ata_device:text}", - "id": 10, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "collapsed": false, - "panels": [], - "repeat": "ata_device", - "repeatDirection": "h" - }, - { - "type": "stat", - "title": "Health", - "description": "", - "id": 11, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 16 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "GOOD", - "color": "green", - "index": 0 - }, - "1": { - "text": "CAUTION", - "color": "orange", - "index": 1 - }, - "2": { - "text": "BAD", - "color": "red", - "index": 2 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - }, - { - "color": "red", - "value": 2 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl:health_code{host=~\"$host\", device=~\"$ata_device\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "{{device}}" - } - ] - }, - { - "type": "stat", - "title": "Temperature", - "description": "", - "id": 12, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 16 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 45 - }, - { - "color": "red", - "value": 55 - } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_temperature{host=~\"$host\", device=~\"$ata_device\", temperature_type=\"current\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "temp" - } - ] - }, - { - "type": "stat", - "title": "Powered On", - "description": "", - "id": 13, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 16 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\", host=~\"$host\", device=~\"$ata_device\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\", host=~\"$host\", device=~\"$ata_device\"} * 3600))", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "hours" - } - ] - }, - { - "type": "stat", - "title": "Capacity", - "description": "", - "id": 14, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 16 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$ata_device\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "bytes" - } - ] - }, - { - "type": "stat", - "title": "Power Cycles", - "description": "", - "id": 15, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 16 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_power_cycle_count{host=~\"$host\", device=~\"$ata_device\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "cycles" - } - ] - }, - { - "type": "stat", - "title": "Reallocated Sectors", - "description": "Attribute 5 raw. Sectors the drive has already retired. Any growth here is the single most predictive sign of impending failure.", - "id": 16, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 16 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_id=\"5\", attribute_value_type=\"raw\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "sectors" - } - ] - }, - { - "type": "stat", - "title": "Pending Sectors", - "description": "Attribute 197 raw. Sectors that failed to read and are waiting to be reallocated on next write. Often precedes reallocations.", - "id": 17, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 16 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_id=\"197\", attribute_value_type=\"raw\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "sectors" - } - ] - }, - { - "type": "table", - "title": "Drive \u2014 $ata_device", - "description": "Identity of this drive. RPM shows SSD when the drive reports no rotation rate.", - "id": 18, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 20 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Capacity" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RPM" - }, - "properties": [ - { - "id": "unit", - "value": "rotrpm" - }, - { - "id": "noValue", - "value": "SSD" - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device{host=~\"$host\", device=~\"$ata_device\"}", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (device) (smartctl_device_rotation_rate{host=~\"$host\", device=~\"$ata_device\"})", - "refId": "B", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$ata_device\"})", - "refId": "C", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "device|model_family|model_name|serial_number|firmware_version|interface|sata_version|form_factor|Value.*" - } - } - }, - { - "id": "joinByField", - "options": { - "byField": "device", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Value #A": true - }, - "indexByName": { - "device": 0, - "model_family": 1, - "model_name": 2, - "serial_number": 3, - "firmware_version": 4, - "interface": 5, - "sata_version": 6, - "form_factor": 7, - "Value #B": 8, - "Value #C": 9 - }, - "renameByName": { - "device": "Device", - "Value #B": "RPM", - "Value #C": "Capacity", - "model_family": "Family", - "model_name": "Model", - "serial_number": "Serial", - "firmware_version": "Firmware", - "interface": "Interface", - "sata_version": "SATA", - "form_factor": "Form" - } - } - } - ] - }, - { - "type": "table", - "title": "SMART Attributes \u2014 $ata_device", - "description": "Every attribute the drive reports, with its raw value. Current/Worst are normalised 0-253 scores (higher is better); Threshold is where the vendor declares failure; Margin is Current minus Threshold. ID sorts as text, so the table is ordered by attribute name.", - "id": 19, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 24 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Margin" - }, - "properties": [ - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 1 - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "green", - "value": 25 - } - ] - } - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background", - "mode": "basic" - } - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"value\"}", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"worst\"})", - "refId": "B", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"thresh\"})", - "refId": "C", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"raw\"})", - "refId": "D", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (attribute_id) (smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"value\"} - on(device, attribute_id) smartctl_device_attribute{host=~\"$host\", device=~\"$ata_device\", attribute_value_type=\"thresh\"})", - "refId": "E", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "attribute_id|attribute_name|Value.*" - } - } - }, - { - "id": "joinByField", - "options": { - "byField": "attribute_id", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "attribute_id": 0, - "attribute_name": 1, - "Value #A": 2, - "Value #B": 3, - "Value #C": 4, - "Value #D": 5, - "Value #E": 6 - }, - "renameByName": { - "attribute_id": "ID", - "attribute_name": "Attribute", - "Value #A": "Current", - "Value #B": "Worst", - "Value #C": "Threshold", - "Value #D": "Raw", - "Value #E": "Margin" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "Attribute", - "desc": false - } - ] - } - } - ] - }, - { - "type": "row", - "title": "${sas_device:text}", - "id": 20, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 - }, - "collapsed": false, - "panels": [], - "repeat": "sas_device", - "repeatDirection": "h" - }, - { - "type": "stat", - "title": "Health", - "description": "", - "id": 21, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 37 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "GOOD", - "color": "green", - "index": 0 - }, - "1": { - "text": "CAUTION", - "color": "orange", - "index": 1 - }, - "2": { - "text": "BAD", - "color": "red", - "index": 2 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - }, - { - "color": "red", - "value": 2 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl:health_code{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "{{device}}" - } - ] - }, - { - "type": "stat", - "title": "Temperature", - "description": "", - "id": 22, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 37 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 45 - }, - { - "color": "red", - "value": 55 - } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_temperature{host=~\"$host\", device=~\"$sas_device\", temperature_type=\"current\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "temp" - } - ] - }, - { - "type": "stat", - "title": "Powered On", - "description": "SAS drives have no ATA attribute 9, so smartctl reports power_on_time from the Background Scan Results log page \u2014 a counter the background-scan subsystem maintains, not the drive's lifetime meter, and one that commonly reads far too low. This tile takes the larger of that counter and the lifetime-hours stamp in the drive's own self-test log, which is why it can exceed what `smartctl -x` prints as 'Accumulated power on time'.", - "id": 23, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 37 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (device) (smartctl_device_power_on_seconds{host=~\"$host\", host=~\"$host\", device=~\"$sas_device\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\", host=~\"$host\", device=~\"$sas_device\"} * 3600))", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "hours" - } - ] - }, - { - "type": "stat", - "title": "Capacity", - "description": "", - "id": 24, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 37 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "bytes" - } - ] - }, - { - "type": "stat", - "title": "Link Speed", - "description": "Negotiated SAS link rate. A drive that has fallen back to a lower rate is worth investigating \u2014 usually a cable or expander.", - "id": 25, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 15, - "y": 37 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bps" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_interface_speed{host=~\"$host\", device=~\"$sas_device\", speed_type=\"current\"} * 1000", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "bps" - } - ] - }, - { - "type": "stat", - "title": "Grown Defects", - "description": "Sectors reallocated since the drive left the factory \u2014 the SAS equivalent of ATA attribute 5. A stable non-zero count is tolerable; growth is not.", - "id": 26, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 37 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_scsi_grown_defect_list{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "defects" - } - ] - }, - { - "type": "stat", - "title": "Uncorrected Errors", - "description": "Errors the drive could not correct with ECC or retries, which means data was lost. Should be zero.", - "id": 27, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 37 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_read_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"} + smartctl_write_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "errors" - } - ] - }, - { - "type": "table", - "title": "Drive \u2014 $sas_device", - "description": "Identity of this drive. SAS drives leave firmware_version empty and report their revision in scsi_revision instead.", - "id": 28, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 41 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Capacity" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RPM" - }, - "properties": [ - { - "id": "unit", - "value": "rotrpm" - }, - { - "id": "noValue", - "value": "SSD" - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (device) (smartctl_device_rotation_rate{host=~\"$host\", device=~\"$sas_device\"})", - "refId": "B", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (device) (smartctl_device_capacity_bytes{host=~\"$host\", device=~\"$sas_device\"})", - "refId": "C", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "device|scsi_vendor|model_name|serial_number|scsi_revision|scsi_version|protocol|form_factor|Value.*" - } - } - }, - { - "id": "joinByField", - "options": { - "byField": "device", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Value #A": true - }, - "indexByName": { - "device": 0, - "scsi_vendor": 1, - "model_name": 2, - "serial_number": 3, - "scsi_revision": 4, - "scsi_version": 5, - "protocol": 6, - "form_factor": 7, - "Value #B": 8, - "Value #C": 9 - }, - "renameByName": { - "device": "Device", - "Value #B": "RPM", - "Value #C": "Capacity", - "scsi_vendor": "Vendor", - "model_name": "Model", - "serial_number": "Serial", - "scsi_revision": "Revision", - "scsi_version": "SCSI", - "protocol": "Bus", - "form_factor": "Form" - } - } - } - ] - }, - { - "type": "timeseries", - "title": "Corrected Errors \u2014 $sas_device", - "description": "Errors the drive fixed on its own. ECC-fast corrections are routine and grow constantly. Delayed corrections and especially rewrites/retries climbing is the signal worth watching.", - "id": 29, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 45 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_read_errors_corrected_by_eccfast{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "read ECC fast" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_read_errors_corrected_by_eccdelayed{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "B", - "instant": false, - "range": true, - "legendFormat": "read ECC delayed" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_read_errors_corrected_by_rereads_rewrites{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "C", - "instant": false, - "range": true, - "legendFormat": "read retries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_write_errors_corrected_by_eccfast{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "D", - "instant": false, - "range": true, - "legendFormat": "write ECC fast" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_write_errors_corrected_by_eccdelayed{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "E", - "instant": false, - "range": true, - "legendFormat": "write ECC delayed" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_write_errors_corrected_by_rereads_rewrites{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "F", - "instant": false, - "range": true, - "legendFormat": "write retries" - } - ] - }, - { - "type": "timeseries", - "title": "Defects & Uncorrected \u2014 $sas_device", - "description": "Flat lines are good. Any upward step here is a real event.", - "id": 30, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 45 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 2, - "fillOpacity": 0, - "showPoints": "auto", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_scsi_grown_defect_list{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "grown defects" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_read_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "B", - "instant": false, - "range": true, - "legendFormat": "read uncorrected" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_write_total_uncorrected_errors{host=~\"$host\", device=~\"$sas_device\"}", - "refId": "C", - "instant": false, - "range": true, - "legendFormat": "write uncorrected" - } - ] - }, - { - "type": "row", - "title": "Self-Tests", - "id": 31, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 53 - }, - "collapsed": false, - "panels": [] - }, - { - "type": "stat", - "title": "Drives Never Tested", - "description": "Drives with no self-test log entries at all. smartd only runs tests if its config includes an -s schedule; merely enabling the service does not.", - "id": 32, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 54 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count(smartctl_device_smart_status{host=~\"$host\"}) - count(count by (device) (smart_selftest_last_lifetime_hours{host=~\"$host\"})) or count(smartctl_device_smart_status{host=~\"$host\"})", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "drives" - } - ] - }, - { - "type": "stat", - "title": "Failed Self-Tests", - "description": "", - "id": 33, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 54 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count(smart_selftest_last_passed{host=~\"$host\"} == 0) or vector(0)", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "tests" - } - ] - }, - { - "type": "stat", - "title": "Oldest Long Test", - "description": "Powered-on hours since the least recently tested drive last completed an extended (long) self-test. Shows 'never run' when no drive has ever completed one \u2014 which is the default state, since smartd schedules no tests unless its config carries an -s directive.", - "id": 34, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 54 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 336 - }, - { - "color": "red", - "value": 1080 - } - ] - }, - "unit": "h", - "noValue": "never run" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{host=~\"$host\", test_type=\"extended\"})", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "hours ago" - } - ] - }, - { - "type": "table", - "title": "Self-Test History", - "description": "Short and extended self-test results per drive, from collectors/smart-selftest.py. 'At Hours' is the drive's powered-on hour count when the test ran, straight from the log. 'Hours Ago' subtracts it from the drive's current powered-on hours; on SAS that figure already falls back to this log, so 0 means the newest log entry is also the highest hour count the drive has ever reported \u2014 not that the test ran just now.", - "id": 35, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 54 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Last Result" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "type": "value", - "options": { - "0": { - "text": "FAILED", - "color": "red", - "index": 0 - }, - "1": { - "text": "passed", - "color": "green", - "index": 1 - } - } - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background", - "mode": "basic" - } - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "label_join(smart_selftest_last_passed{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\")", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (drive_test) (label_join(smart_selftest_last_lifetime_hours{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))", - "refId": "B", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (drive_test) (label_join(smart_selftest_count{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))", - "refId": "C", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (drive_test) (label_join(max by (device) (smartctl_device_power_on_seconds{host=~\"$host\"} or (smart_selftest_last_lifetime_hours{host=~\"$host\"} * 3600)) / 3600 - on(device) group_right() smart_selftest_last_lifetime_hours{host=~\"$host\"}, \"drive_test\", \"/\", \"device\", \"test_type\"))", - "refId": "D", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "drive_test|device|test_type|Value.*" - } - } - }, - { - "id": "joinByField", - "options": { - "byField": "drive_test", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "drive_test": true - }, - "indexByName": { - "device": 0, - "test_type": 1, - "Value #A": 2, - "Value #B": 3, - "Value #C": 4, - "Value #D": 5 - }, - "renameByName": { - "device": "Device", - "test_type": "Test", - "Value #A": "Last Result", - "Value #B": "At Hours", - "Value #C": "Logged", - "Value #D": "Hours Ago" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "Device", - "desc": false - } - ] - } - } - ] - }, - { - "type": "row", - "title": "Attribute Trends", - "id": 41, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 64 - }, - "collapsed": true, - "panels": [ - { - "type": "timeseries", - "title": "Reallocated Sectors (all SATA drives)", - "description": "", - "id": 36, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 65 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 2, - "fillOpacity": 0, - "showPoints": "auto", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=\"5\", attribute_value_type=\"raw\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{device}}" - } - ] - }, - { - "type": "timeseries", - "title": "Pending & Uncorrectable Sectors", - "description": "", - "id": 37, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 65 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 2, - "fillOpacity": 0, - "showPoints": "auto", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=~\"197|198\", attribute_value_type=\"raw\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{device}} {{attribute_name}}" - } - ] - }, - { - "type": "timeseries", - "title": "CRC Errors (cabling)", - "description": "Attribute 199 counts errors on the SATA link rather than the platters. Growth here usually means a cable or backplane, not a dying disk.", - "id": 38, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 65 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 2, - "fillOpacity": 0, - "showPoints": "auto", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_attribute{host=~\"$host\", attribute_id=\"199\", attribute_value_type=\"raw\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{device}}" - } - ] - }, - { - "type": "timeseries", - "title": "Data Written / Read (SAS)", - "description": "", - "id": 39, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 72 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes", - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_bytes_written{host=~\"$host\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{device}} written" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_bytes_read{host=~\"$host\"}", - "refId": "B", - "instant": false, - "range": true, - "legendFormat": "{{device}} read" - } - ] - }, - { - "type": "timeseries", - "title": "Health Code History", - "description": "", - "id": 40, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 72 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "lineWidth": 2, - "fillOpacity": 0, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "GOOD", - "color": "green", - "index": 0 - }, - "1": { - "text": "CAUTION", - "color": "orange", - "index": 1 - }, - "2": { - "text": "BAD", - "color": "red", - "index": 2 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0, - "max": 2 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl:health_code{host=~\"$host\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{device}}" - } - ] - } - ] - }, - { - "type": "row", - "title": "Collector Health", - "id": 45, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 65 - }, - "collapsed": true, - "panels": [ - { - "type": "timeseries", - "title": "smartctl Exit Status", - "description": "Non-zero means smartctl itself had trouble reading the device. Bit 0 is a command-line problem, bit 1 a device-open failure, bit 3 a failing drive.", - "id": 42, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 66 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_device_smartctl_exit_status{host=~\"$host\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{device}}" - } - ] - }, - { - "type": "timeseries", - "title": "Self-Test Collector Duration", - "description": "", - "id": 43, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 66 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s", - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smart_selftest_collection_duration_seconds{host=~\"$host\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "collection" - } - ] - }, - { - "type": "timeseries", - "title": "Devices Seen", - "description": "These should agree. A divergence means one of the two cannot see some drives.", - "id": 44, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 66 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smartctl_devices{host=~\"$host\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "exporter" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "smart_selftest_devices{host=~\"$host\"}", - "refId": "B", - "instant": false, - "range": true, - "legendFormat": "self-test collector" - } - ] - } - ] - } - ] -} diff --git a/dashboards/zfs-pools.json b/dashboards/zfs-pools.json deleted file mode 100644 index c36a42a..0000000 --- a/dashboards/zfs-pools.json +++ /dev/null @@ -1,4395 +0,0 @@ -{ - "title": "ZFS Pools", - "uid": "zfs-pools", - "description": "ZFS capacity, health, integrity, snapshot retention, dataset properties and performance. Fed by node_exporter's zfs and systemd collectors plus collectors/zfs-metrics.py from lgtm.nix.", - "tags": [ - "zfs", - "storage" - ], - "timezone": "browser", - "schemaVersion": 41, - "version": 1, - "editable": true, - "graphTooltip": 1, - "refresh": "1m", - "time": { - "from": "now-24h", - "to": "now" - }, - "templating": { - "list": [ - { - "name": "host", - "label": "Host", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "query": { - "qryType": 1, - "query": "label_values(zfs_pool_health_code, host)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "includeAll": false, - "multi": false, - "options": [], - "sort": 1, - "hide": 0, - "regex": "" - }, - { - "name": "pool", - "label": "Pool", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "query": { - "qryType": 1, - "query": "label_values(zfs_pool_health_code{host=~\"$host\"}, pool)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "includeAll": true, - "multi": true, - "options": [], - "sort": 1, - "hide": 0, - "regex": "", - "allValue": ".*", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - } - } - ] - }, - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "panels": [ - { - "type": "row", - "title": "Pool Overview", - "id": 1, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "collapsed": false, - "panels": [] - }, - { - "type": "stat", - "title": "Pool Health", - "description": "Health as reported by `zpool list -o health`.", - "id": 2, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "ONLINE", - "color": "green", - "index": 0 - }, - "1": { - "text": "DEGRADED", - "color": "orange", - "index": 1 - }, - "2": { - "text": "FAULTED", - "color": "red", - "index": 2 - }, - "3": { - "text": "OFFLINE", - "color": "text", - "index": 3 - }, - "4": { - "text": "UNAVAIL", - "color": "red", - "index": 4 - }, - "5": { - "text": "REMOVED", - "color": "purple", - "index": 5 - }, - "6": { - "text": "SUSPENDED", - "color": "red", - "index": 6 - }, - "7": { - "text": "AVAIL", - "color": "blue", - "index": 7 - }, - "8": { - "text": "UNKNOWN", - "color": "text", - "index": 8 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_health_code{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "{{pool}}" - } - ] - }, - { - "type": "stat", - "title": "Data Errors", - "description": "Permanent, unrepairable errors from `zpool status`. Anything above zero means data has been lost and the affected files need restoring.", - "id": 3, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(zfs_pool_data_errors{host=~\"$host\", pool=~\"$pool\"}) or vector(0)", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "errors" - } - ] - }, - { - "type": "stat", - "title": "vdev Errors", - "description": "Total read + write + checksum errors across all vdevs. Cleared by `zpool clear`.", - "id": 4, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(zfs_pool_vdev_errors_total{host=~\"$host\", pool=~\"$pool\"}) or vector(0)", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "errors" - } - ] - }, - { - "type": "stat", - "title": "Last Scrub", - "description": "Time since the most recent completed scrub. Turns amber past ~5 weeks, which is a monthly scrub schedule having missed a run.", - "id": 5, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3024000 - }, - { - "color": "red", - "value": 6048000 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "time() - max(zfs_pool_last_scrub_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "ago" - } - ] - }, - { - "type": "stat", - "title": "Scrub / Resilver", - "description": "", - "id": 6, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "idle", - "color": "green", - "index": 0 - }, - "1": { - "text": "RUNNING", - "color": "blue", - "index": 1 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(zfs_pool_scrub_in_progress{host=~\"$host\", pool=~\"$pool\"})", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "scrub" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(zfs_pool_resilver_in_progress{host=~\"$host\", pool=~\"$pool\"})", - "refId": "B", - "instant": true, - "range": false, - "legendFormat": "resilver" - } - ] - }, - { - "type": "stat", - "title": "Locked Datasets", - "description": "Encrypted datasets whose key is not loaded. These are unreadable and are silently skipped by anything that walks the filesystem, including backups.", - "id": 7, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count(zfs_dataset_encrypted{host=~\"$host\", pool=~\"$pool\"} == 1 and zfs_dataset_key_available{host=~\"$host\", pool=~\"$pool\"} == 0) or vector(0)", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "locked" - } - ] - }, - { - "type": "gauge", - "title": "Space Used", - "description": "How full the pool is: allocated / total. ZFS write performance degrades sharply past ~80%, and the pool becomes effectively read-only near 100%. The byte figures behind this are in 'Capacity by Pool' below.", - "id": 8, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 0, - "y": 5 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0, - "max": 1, - "unit": "percentunit", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.8 - }, - { - "color": "red", - "value": 0.9 - } - ] - } - }, - "overrides": [] - }, - "options": { - "showThresholdLabels": false, - "showThresholdMarkers": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_capacity_ratio{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "{{pool}}" - } - ] - }, - { - "type": "gauge", - "title": "Fragmentation", - "description": "Free-space fragmentation.", - "id": 9, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 5, - "y": 5 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0, - "max": 1, - "unit": "percentunit", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "red", - "value": 0.75 - } - ] - } - }, - "overrides": [] - }, - "options": { - "showThresholdLabels": false, - "showThresholdMarkers": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "{{pool}}" - } - ] - }, - { - "type": "stat", - "title": "Free Space", - "description": "", - "id": 10, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 10, - "y": 5 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "{{pool}}" - } - ] - }, - { - "type": "stat", - "title": "Snapshots Overdue", - "description": "Dataset/label pairs whose newest snapshot is more than two intervals old \u2014 i.e. the auto-snapshot timer has missed at least one run.", - "id": 11, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 15, - "y": 5 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count((time() - zfs_snapshot_newest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"}) > on(label) group_left() (2 * zfs_snapshot_interval_seconds{host=~\"$host\"})) or vector(0)", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "overdue" - } - ] - }, - { - "type": "stat", - "title": "Retention Depth", - "description": "Age of the oldest snapshot on the pool \u2014 how far back you can actually restore from.", - "id": 12, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 19, - "y": 5 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "oldest" - } - ] - }, - { - "type": "timeseries", - "title": "Pool Space Used / Free", - "description": "Stacked, so the top of each pool's band is its total size and the split between the bands is the usage fraction the gauge above shows.", - "id": 13, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 40, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes", - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_allocated_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{pool}} used" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "B", - "instant": false, - "range": true, - "legendFormat": "{{pool}} free" - } - ] - }, - { - "type": "timeseries", - "title": "Fragmentation", - "description": "Free-space fragmentation \u2014 how scattered the pool's remaining free space is, not how scattered the data is. It rises as a pool fills and stays high afterwards; only rewriting data (send/recv, or deleting a lot) brings it down. Kept on its own axis because it is unrelated to space used, and plotting the two together invites reading one as the other.", - "id": 14, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit", - "min": 0, - "max": 1 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{pool}}" - } - ] - }, - { - "type": "table", - "title": "Capacity by Pool", - "description": "Per-pool space in bytes. 'Size' is the raw pool size zpool reports, which for raidz is before parity overhead \u2014 so 'Free' is the number to trust for how much you can still write. 'Snapshots' is space held only by snapshots.", - "id": 15, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 17 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "^(Size|Used|Free|Snapshots)$" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Used %" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "max", - "value": 1 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.8 - }, - { - "color": "red", - "value": 0.9 - } - ] - } - }, - { - "id": "custom.cellOptions", - "value": { - "type": "gauge", - "mode": "gradient" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Fragmentation" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_size_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (pool) (zfs_pool_allocated_bytes{host=~\"$host\", pool=~\"$pool\"})", - "refId": "B", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (pool) (zfs_pool_free_bytes{host=~\"$host\", pool=~\"$pool\"})", - "refId": "C", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (pool) (zfs_pool_capacity_ratio{host=~\"$host\", pool=~\"$pool\"})", - "refId": "D", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (pool) (zfs_pool_fragmentation_ratio{host=~\"$host\", pool=~\"$pool\"})", - "refId": "E", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (pool) (zfs_pool_snapshot_used_bytes{host=~\"$host\", pool=~\"$pool\"})", - "refId": "F", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (pool) (zfs_pool_dedup_ratio{host=~\"$host\", pool=~\"$pool\"})", - "refId": "G", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "pool|Value.*" - } - } - }, - { - "id": "joinByField", - "options": { - "byField": "pool", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "pool": 0, - "Value #A": 1, - "Value #B": 2, - "Value #C": 3, - "Value #D": 4, - "Value #E": 5, - "Value #F": 6, - "Value #G": 7 - }, - "renameByName": { - "pool": "Pool", - "Value #A": "Size", - "Value #B": "Used", - "Value #C": "Free", - "Value #D": "Used %", - "Value #E": "Fragmentation", - "Value #F": "Snapshots", - "Value #G": "Dedup" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "Used %", - "desc": true - } - ] - } - } - ] - }, - { - "type": "row", - "title": "Integrity & Errors", - "id": 16, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "collapsed": false, - "panels": [] - }, - { - "type": "table", - "title": "vdev State", - "description": "Every vdev and leaf device in the selected pool(s).", - "id": 17, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 26 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "ONLINE", - "color": "green", - "index": 0 - }, - "1": { - "text": "DEGRADED", - "color": "orange", - "index": 1 - }, - "2": { - "text": "FAULTED", - "color": "red", - "index": 2 - }, - "3": { - "text": "OFFLINE", - "color": "text", - "index": 3 - }, - "4": { - "text": "UNAVAIL", - "color": "red", - "index": 4 - }, - "5": { - "text": "REMOVED", - "color": "purple", - "index": 5 - }, - "6": { - "text": "SUSPENDED", - "color": "red", - "index": 6 - }, - "7": { - "text": "AVAIL", - "color": "blue", - "index": 7 - }, - "8": { - "text": "UNKNOWN", - "color": "text", - "index": 8 - } - } - } - ], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "State" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-background", - "mode": "basic" - } - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_vdev_state_code{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "pool|vdev|Value" - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "pool": "Pool", - "vdev": "vdev", - "Value": "State" - } - } - } - ] - }, - { - "type": "timeseries", - "title": "vdev Error Counters", - "description": "Cumulative per-vdev errors. A counter that climbs is a disk on its way out.", - "id": 18, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 26 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 2, - "fillOpacity": 0, - "showPoints": "auto", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_vdev_errors_total{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{pool}}/{{vdev}} {{type}}" - } - ] - }, - { - "type": "table", - "title": "Scrub & Resilver History", - "description": "Result of the last completed scrub per pool. Errors here are blocks that could not be repaired from redundancy.", - "id": 19, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 35 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Scrub Finished" - }, - "properties": [ - { - "id": "unit", - "value": "dateTimeAsIso" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Resilver Finished" - }, - "properties": [ - { - "id": "unit", - "value": "dateTimeAsIso" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Repaired" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Errors" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-background", - "mode": "basic" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_last_scrub_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"} * 1000", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_last_scrub_duration_seconds{host=~\"$host\", pool=~\"$pool\"}", - "refId": "B", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_last_scrub_errors{host=~\"$host\", pool=~\"$pool\"}", - "refId": "C", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_last_scrub_repaired_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "D", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_last_resilver_end_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"} * 1000", - "refId": "E", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "pool|Value.*" - } - } - }, - { - "id": "joinByField", - "options": { - "byField": "pool", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "pool": "Pool", - "Value #A": "Scrub Finished", - "Value #B": "Duration", - "Value #C": "Errors", - "Value #D": "Repaired", - "Value #E": "Resilver Finished" - } - } - } - ] - }, - { - "type": "timeseries", - "title": "Pool Data Errors", - "description": "Permanent data errors over time.", - "id": 20, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 35 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "auto", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_data_errors{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{pool}}" - } - ] - }, - { - "type": "row", - "title": "Snapshots & Retention", - "id": 21, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 42 - }, - "collapsed": false, - "panels": [] - }, - { - "type": "bargauge", - "title": "Time Since Last Snapshot", - "description": "Age of the newest snapshot of each label. Compare against the label's own cadence \u2014 'frequent' is every 15 minutes, 'monthly' every month, so the colour thresholds here are only a coarse hint.", - "id": 22, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 43 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 7200 - }, - { - "color": "red", - "value": 172800 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "displayMode": "basic", - "orientation": "horizontal", - "showUnfilled": true, - "valueMode": "color", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "{{label}}" - } - ] - }, - { - "type": "bargauge", - "title": "Snapshots vs Retention Target", - "description": "Worst dataset's snapshot count as a fraction of the configured services.zfs.autoSnapshot retention. 100% means the full set is on disk; a freshly-created dataset legitimately sits below target until it fills up.", - "id": 23, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 43 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "green", - "value": 0.9 - } - ] - }, - "unit": "percentunit", - "max": 1.2 - }, - "overrides": [] - }, - "options": { - "displayMode": "basic", - "orientation": "horizontal", - "showUnfilled": true, - "valueMode": "color", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "min by (label) (zfs_snapshot_count{host=~\"$host\", pool=~\"$pool\"} / on(label) group_left() zfs_snapshot_retention_target{host=~\"$host\"})", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "{{label}}" - } - ] - }, - { - "type": "bargauge", - "title": "Oldest Snapshot", - "description": "How far back each label reaches.", - "id": 24, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 43 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "displayMode": "basic", - "orientation": "horizontal", - "showUnfilled": true, - "valueMode": "color", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "time() - min by (label) (zfs_pool_snapshot_oldest_timestamp_seconds{host=~\"$host\", pool=~\"$pool\"})", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "{{label}}" - } - ] - }, - { - "type": "timeseries", - "title": "Snapshot Count by Label", - "description": "Total snapshots per label. A steady sawtooth is healthy rotation; a monotonic decline means snapshots are expiring but not being replaced.", - "id": 25, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 51 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (label) (zfs_snapshot_count{host=~\"$host\", pool=~\"$pool\"})", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{label}}" - } - ] - }, - { - "type": "timeseries", - "title": "Snapshot Space by Label", - "description": "Space held down by snapshots \u2014 the cost of your retention policy.", - "id": 26, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 51 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 40, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes", - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (label) (zfs_pool_snapshot_used_bytes{host=~\"$host\", pool=~\"$pool\"})", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{label}}" - } - ] - }, - { - "type": "stat", - "title": "Snapshot / Scrub Job Failures", - "description": "systemd units for the auto-snapshot and auto-scrub timers currently in a failed state.", - "id": 27, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 59 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "textMode": "auto", - "wideLayout": true, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(node_systemd_unit_state{host=~\"$host\", name=~\"zfs-(snapshot|scrub).*\", state=\"failed\"}) or vector(0)", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "failed units" - } - ] - }, - { - "type": "bargauge", - "title": "Time Since Timer Last Fired", - "description": "When each zfs-* systemd timer last triggered. This is the job side of the picture; the snapshot ages above are the result side.", - "id": 28, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 6, - "w": 18, - "x": 6, - "y": 59 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "displayMode": "basic", - "orientation": "horizontal", - "showUnfilled": true, - "valueMode": "color", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "time() - node_systemd_timer_last_trigger_seconds{host=~\"$host\", name=~\"zfs-.*\"}", - "refId": "A", - "instant": true, - "range": false, - "legendFormat": "{{name}}" - } - ] - }, - { - "type": "row", - "title": "Datasets", - "id": 29, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 65 - }, - "collapsed": false, - "panels": [] - }, - { - "type": "timeseries", - "title": "Dataset Usage Over Time", - "description": "Space each dataset is responsible for, including its snapshots and descendants \u2014 so a parent's line contains its children's. Unstacked and unfilled so a single dataset's growth is readable against the others rather than riding on top of them. The legend carries last and max per dataset; the instant breakdown is in 'Capacity by Dataset' below.", - "id": 30, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 66 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 2, - "fillOpacity": 0, - "showPoints": "auto", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes", - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_dataset_used_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{dataset}}" - } - ] - }, - { - "type": "table", - "title": "Capacity by Dataset", - "description": "Per-dataset space, separate from the per-pool figures above. 'Used' is the total a dataset is responsible for; 'Own Data' and 'Snapshots' are the two halves of that. 'Logical' is the size before compression, so Logical / Referenced is where the compression ratio comes from. 'Available' is shared with every sibling dataset unless a quota says otherwise.", - "id": 31, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 75 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "^(Used|Own Data|Snapshots|Referenced|Logical|Available|Quota)$" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Quota" - }, - "properties": [ - { - "id": "noValue", - "value": "none" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Ratio" - }, - "properties": [ - { - "id": "decimals", - "value": 2 - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_dataset_used_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_used_by_dataset_bytes{host=~\"$host\", pool=~\"$pool\"})", - "refId": "B", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_used_by_snapshots_bytes{host=~\"$host\", pool=~\"$pool\"})", - "refId": "C", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_referenced_bytes{host=~\"$host\", pool=~\"$pool\"})", - "refId": "D", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_logical_used_bytes{host=~\"$host\", pool=~\"$pool\"})", - "refId": "E", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_available_bytes{host=~\"$host\", pool=~\"$pool\"})", - "refId": "F", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_compression_ratio{host=~\"$host\", pool=~\"$pool\"})", - "refId": "G", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max by (dataset) (zfs_dataset_quota_bytes{host=~\"$host\", pool=~\"$pool\"} > 0)", - "refId": "H", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "pool|dataset|Value.*" - } - } - }, - { - "id": "joinByField", - "options": { - "byField": "dataset", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "pool": 0, - "dataset": 1, - "Value #A": 2, - "Value #B": 3, - "Value #C": 4, - "Value #D": 5, - "Value #E": 6, - "Value #F": 7, - "Value #G": 8, - "Value #H": 9 - }, - "renameByName": { - "dataset": "Dataset", - "pool": "Pool", - "Value #A": "Used", - "Value #B": "Own Data", - "Value #C": "Snapshots", - "Value #D": "Referenced", - "Value #E": "Logical", - "Value #F": "Available", - "Value #G": "Ratio", - "Value #H": "Quota" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "Used", - "desc": true - } - ] - } - } - ] - }, - { - "type": "table", - "title": "Compression", - "description": "group_left() attaches the algorithm label from the info metric onto the ratio, so this needs only one query and no join transformation.", - "id": 32, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 83 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_dataset_compression_ratio{host=~\"$host\", pool=~\"$pool\"} * on(pool, dataset) group_left(compression) zfs_dataset_compression{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "pool|dataset|compression|Value" - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "pool": 0, - "dataset": 1, - "compression": 2, - "Value": 3 - }, - "renameByName": { - "pool": "Pool", - "dataset": "Dataset", - "compression": "Algorithm", - "Value": "Ratio" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "Ratio", - "desc": true - } - ] - } - } - ] - }, - { - "type": "table", - "title": "Encryption", - "description": "Encryption suite per dataset and the state of its key. Datasets that are not encrypted show suite 'off' and a dash \u2014 only an encrypted dataset can be locked.", - "id": 33, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 83 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "\u2014", - "color": "text", - "index": 0 - }, - "1": { - "text": "unlocked", - "color": "green", - "index": 1 - }, - "2": { - "text": "LOCKED", - "color": "red", - "index": 2 - }, - "3": { - "text": "unknown", - "color": "orange", - "index": 3 - } - } - } - ], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Key" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-background", - "mode": "basic" - } - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_dataset_keystatus_code{host=~\"$host\", pool=~\"$pool\"} * on(pool, dataset) group_left(encryption) zfs_dataset_encryption{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "pool|dataset|encryption|Value" - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": { - "pool": 0, - "dataset": 1, - "encryption": 2, - "Value": 3 - }, - "renameByName": { - "pool": "Pool", - "dataset": "Dataset", - "encryption": "Suite", - "Value": "Key" - } - } - } - ] - }, - { - "type": "row", - "title": "Performance", - "id": 34, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 91 - }, - "collapsed": false, - "panels": [] - }, - { - "type": "timeseries", - "title": "Dataset Throughput", - "description": "Per-dataset throughput from node_exporter's objset kstats. Writes are drawn negative.", - "id": 35, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 92 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(node_zfs_zpool_dataset_nread{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{zpool}}/{{dataset}} read" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "-rate(node_zfs_zpool_dataset_nwritten{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", - "refId": "B", - "instant": false, - "range": true, - "legendFormat": "{{zpool}}/{{dataset}} write" - } - ] - }, - { - "type": "timeseries", - "title": "Dataset IOPS", - "description": "Writes are drawn negative.", - "id": 36, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 92 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "iops" - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(node_zfs_zpool_dataset_reads{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{zpool}}/{{dataset}} read" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "-rate(node_zfs_zpool_dataset_writes{host=~\"$host\", zpool=~\"$pool\"}[$__rate_interval])", - "refId": "B", - "instant": false, - "range": true, - "legendFormat": "{{zpool}}/{{dataset}} write" - } - ] - }, - { - "type": "timeseries", - "title": "ARC Size", - "description": "", - "id": 37, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 100 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes", - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_zfs_arc_size{host=~\"$host\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "size" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_zfs_arc_c_max{host=~\"$host\"}", - "refId": "B", - "instant": false, - "range": true, - "legendFormat": "target max (c_max)" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_zfs_arc_c_min{host=~\"$host\"}", - "refId": "C", - "instant": false, - "range": true, - "legendFormat": "target min (c_min)" - } - ] - }, - { - "type": "timeseries", - "title": "ARC Hit Ratio", - "description": "clamp_min avoids a divide-by-zero while the pool is idle.", - "id": 38, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 100 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit", - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval]) / clamp_min(rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval]) + rate(node_zfs_arc_misses{host=~\"$host\"}[$__rate_interval]), 1)", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "hit ratio" - } - ] - }, - { - "type": "timeseries", - "title": "ARC Hits / Misses", - "description": "", - "id": 39, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 100 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops", - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(node_zfs_arc_hits{host=~\"$host\"}[$__rate_interval])", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "hits" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(node_zfs_arc_misses{host=~\"$host\"}[$__rate_interval])", - "refId": "B", - "instant": false, - "range": true, - "legendFormat": "misses" - } - ] - }, - { - "type": "row", - "title": "Pool Internals", - "id": 44, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 108 - }, - "collapsed": true, - "panels": [ - { - "type": "timeseries", - "title": "Reclaim & Leaks", - "description": "`freeing` is space still being reclaimed in the background after a delete. `leaked` should always be zero.", - "id": 40, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 109 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes", - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_freeing_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{pool}} freeing" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_leaked_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "B", - "instant": false, - "range": true, - "legendFormat": "{{pool}} leaked" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_checkpoint_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "C", - "instant": false, - "range": true, - "legendFormat": "{{pool}} checkpoint" - } - ] - }, - { - "type": "timeseries", - "title": "Dedup Ratio", - "description": "1.00 unless deduplication is enabled, which it usually should not be.", - "id": 41, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 109 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_dedup_ratio{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{pool}}" - } - ] - }, - { - "type": "timeseries", - "title": "Pool Health History", - "description": "Health code over time \u2014 useful for spotting a pool that briefly degraded and recovered. zfs_pool_scan_state is also collected if you want to plot scrub/resilver phases alongside it.", - "id": 42, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 109 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "lineWidth": 2, - "fillOpacity": 0, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "ONLINE", - "color": "green", - "index": 0 - }, - "1": { - "text": "DEGRADED", - "color": "orange", - "index": 1 - }, - "2": { - "text": "FAULTED", - "color": "red", - "index": 2 - }, - "3": { - "text": "OFFLINE", - "color": "text", - "index": 3 - }, - "4": { - "text": "UNAVAIL", - "color": "red", - "index": 4 - }, - "5": { - "text": "REMOVED", - "color": "purple", - "index": 5 - }, - "6": { - "text": "SUSPENDED", - "color": "red", - "index": 6 - }, - "7": { - "text": "AVAIL", - "color": "blue", - "index": 7 - }, - "8": { - "text": "UNKNOWN", - "color": "text", - "index": 8 - } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0, - "max": 8 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_pool_health_code{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{pool}}" - } - ] - }, - { - "type": "table", - "title": "Dataset Properties", - "description": "A quota of 0 means unlimited. Auto-snapshot reflects the com.sun:auto-snapshot property that zfs-auto-snapshot honours.", - "id": 43, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 116 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Record Size" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Quota" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Ref Quota" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Auto-snapshot" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "type": "value", - "options": { - "0": { - "text": "no", - "color": "orange", - "index": 0 - }, - "1": { - "text": "yes", - "color": "green", - "index": 1 - } - } - } - ] - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_dataset_recordsize_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "A", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_dataset_quota_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "B", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_dataset_refquota_bytes{host=~\"$host\", pool=~\"$pool\"}", - "refId": "C", - "instant": true, - "range": false, - "format": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_dataset_autosnapshot_enabled{host=~\"$host\", pool=~\"$pool\"}", - "refId": "D", - "instant": true, - "range": false, - "format": "table" - } - ], - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "pattern": "dataset|Value.*" - } - } - }, - { - "id": "joinByField", - "options": { - "byField": "dataset", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "dataset": "Dataset", - "Value #A": "Record Size", - "Value #B": "Quota", - "Value #C": "Ref Quota", - "Value #D": "Auto-snapshot" - } - } - } - ] - } - ] - }, - { - "type": "row", - "title": "Collector Health", - "id": 48, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 109 - }, - "collapsed": true, - "panels": [ - { - "type": "timeseries", - "title": "ZFS Collector Duration", - "description": "How long collectors/zfs-metrics.py takes. Listing every snapshot is the slow part; if this approaches the 1 minute timer interval, raise OnUnitActiveSec.", - "id": 45, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 110 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s", - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "zfs_metrics_collection_duration_seconds{host=~\"$host\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "collection" - } - ] - }, - { - "type": "timeseries", - "title": "Textfile Age", - "description": "Age of each .prom file. If this climbs without bound the collector has stopped writing and every ZFS metric above is stale.", - "id": 46, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 110 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s", - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "time() - node_textfile_mtime_seconds{host=~\"$host\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "{{file}}" - } - ] - }, - { - "type": "timeseries", - "title": "Scrape Health", - "description": "node_textfile_scrape_error goes to 1 if a .prom file is malformed, in which case node_exporter drops it entirely.", - "id": 47, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 110 - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "lineWidth": 1, - "fillOpacity": 15, - "showPoints": "never", - "spanNulls": true, - "axisBorderShow": false, - "axisCenteredZero": false, - "gradientMode": "none", - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "up{job=\"node\"}", - "refId": "A", - "instant": false, - "range": true, - "legendFormat": "node exporter up" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_textfile_scrape_error{host=~\"$host\"}", - "refId": "B", - "instant": false, - "range": true, - "legendFormat": "textfile parse error" - } - ] - } - ] - } - ] -} @@ -29,68 +29,73 @@ in collectors // { default = collectors.zfs-metrics; } ); - checks = forAllSystems (pkgs: { - # promtool over the recording rules. The server module also runs this at build - # time via services.prometheus.checkConfig, but that only fires for people who - # actually deploy it. - prometheus-rules = pkgs.runCommand "check-prometheus-rules" - { nativeBuildInputs = [ pkgs.prometheus.cli ]; } - '' - promtool check rules ${./rules}/*.yml - touch "$out" - ''; + checks = forAllSystems (pkgs: + let + dashboardsForCheck = pkgs.linkFarm "dashboards" (pkgs.lib.mapAttrsToList + # The .json suffix matters: the check globs *.json, and a + # linkFarm named without it silently matches nothing. + (name: drv: { name = "${name}.json"; path = drv; }) + # callPackages, not callPackage: the latter adds override/ + # overrideDerivation attrs that are functions, not derivations. + (pkgs.callPackages ./modules/dashboards.nix { + dsUid = { prometheus = "prometheus"; loki = "loki"; }; + })); + in + { + # promtool over the recording rules. The server module also runs this at build + # time via services.prometheus.checkConfig, but that only fires for people who + # actually deploy it. + prometheus-rules = pkgs.runCommand "check-prometheus-rules" + { nativeBuildInputs = [ pkgs.prometheus.cli ]; } + '' + promtool check rules ${./rules}/*.yml + touch "$out" + ''; - # The dashboard JSON is generated by the scripts next to it and committed, so it - # can rot silently if someone edits a generator without re-running it. Regenerate - # here and diff. - dashboards-current = pkgs.runCommand "check-dashboards-current" - { nativeBuildInputs = [ pkgs.python3 pkgs.diffutils ]; } - '' - cp -r ${./dashboards} work - chmod -R u+w work - cd work + # An empty or malformed dashboard does not fail any build on its own -- it + # provisions fine and simply renders nothing -- so assert on the content. + dashboards-nonempty = pkgs.runCommand "check-dashboards-nonempty" + { nativeBuildInputs = [ pkgs.jq ]; } + '' + shopt -s nullglob + found=0 + for f in ${dashboardsForCheck}/*.json; do + found=$((found + 1)) + name=$(basename "$f") + [ -s "$f" ] || { echo "$name is empty"; exit 1; } + jq -e '.title and .uid and (.panels | length > 0)' "$f" >/dev/null \ + || { echo "$name has no title, uid or panels"; exit 1; } + echo "$name: $(jq -r '"\(.title) — \(.panels | length) panels"' "$f")" + done + # A glob that matches nothing would otherwise make this check vacuous. + [ "$found" -eq 4 ] || { echo "expected 4 dashboards, found $found"; exit 1; } + touch "$out" + ''; - # Note the loop variable is not called "out": that is the derivation's own - # output path, and shadowing it makes the final touch silently write - # somewhere else and the build fail with "failed to produce output path". - check() { - local generator="$1" committed="$2" - python3 "$generator" > regenerated.json - if ! diff -u "$committed" regenerated.json; then - echo - echo "dashboards/$committed is stale." - echo "Re-run: python3 dashboards/$generator > dashboards/$committed" - exit 1 - fi - } - - check gen_smart.py smart.json - check gen_zfs.py zfs-pools.json - touch "$out" - ''; - - # Both halves have to evaluate, separately and together. Catches option typos - # and missing mkIf guards without needing a real machine. - modules-eval = (nixpkgs.lib.nixosSystem { - system = pkgs.stdenv.hostPlatform.system; - modules = [ - self.nixosModules.default - ({ lib, ... }: { - services.telemetry.collectors.enable = true; - services.telemetry.server = { - enable = true; - collectorHosts = [ "one" "two" ]; - proxy.enable = true; - # Never read during evaluation; only has to satisfy the assertion. - grafana.secretKeyFile = "/run/secrets/grafana-secret-key"; - }; - boot.loader.grub.devices = [ "/dev/sda" ]; - fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; }; - system.stateVersion = "25.11"; - }) - ]; - }).config.system.build.toplevel; - }); + # Both halves have to evaluate, separately and together. Catches option typos + # and missing mkIf guards without needing a real machine -- and because the + # dashboards are generated during the build, this also runs both generators and + # fails if either one crashes or emits something jq cannot parse. + modules-eval = (nixpkgs.lib.nixosSystem { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + self.nixosModules.default + ({ lib, ... }: { + services.telemetry.collectors.enable = true; + services.telemetry.server = { + enable = true; + collectorHosts = [ "one" "two" ]; + proxy.enable = true; + # Never read during evaluation; only has to satisfy the assertion. + grafana.secretKeyFile = "/run/secrets/grafana-secret-key"; + }; + boot.loader.grub.devices = [ "/dev/sda" ]; + fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; }; + system.stateVersion = "25.11"; + }) + ]; + }).config.system.build.toplevel; + }); devShells = forAllSystems (pkgs: { default = pkgs.mkShell { diff --git a/modules/dashboards.nix b/modules/dashboards.nix index 125e3e2..2fc24c8 100644 --- a/modules/dashboards.nix +++ b/modules/dashboards.nix @@ -2,7 +2,7 @@ # # Returns an attrset of { name = derivation; }, where each derivation is a single # dashboard JSON file. modules/server.nix drops these into /etc/grafana-dashboards. -{ lib, runCommand, fetchurl, jq, dsUid }: +{ lib, runCommand, fetchurl, jq, python3, dsUid }: let # Dashboards exported from grafana.com carry an `__inputs` block and reference their @@ -29,12 +29,23 @@ let | .id = null ''; + # One jq invocation, shared by both sources of dashboards below. + # + # Deliberately a single line with no trailing newline: callers append redirections + # (`< $src > "$out"`), and a multi-line string would put those on a line of their own + # as a separate command -- which truncates $out to zero bytes and sends jq's output to + # the build log, without failing the build. + jqFixup = lib.concatStringsSep " " [ + "jq" + "--arg prom ${lib.escapeShellArg dsUid.prometheus}" + "--arg loki ${lib.escapeShellArg dsUid.loki}" + (lib.escapeShellArg fixupProgram) + ]; + fixup = name: src: runCommand "grafana-dashboard-${name}.json" { nativeBuildInputs = [ jq ]; } '' - jq --arg prom ${lib.escapeShellArg dsUid.prometheus} \ - --arg loki ${lib.escapeShellArg dsUid.loki} \ - ${lib.escapeShellArg fixupProgram} < ${src} > "$out" + ${jqFixup} < ${src} > "$out" ''; fromGrafanaCom = { name, id, rev, hash }: @@ -42,6 +53,28 @@ let url = "https://grafana.com/api/dashboards/${toString id}/revisions/${toString rev}/download"; inherit hash; }); + + # The generators and the panel helpers they import. + generators = ../dashboards; + + # Our own dashboards are generated here rather than committed as JSON. The JSON is + # machine-written and the generator beside it is the only source of truth, so keeping + # both in git only creates a way for them to drift -- and a build that runs the + # generator cannot serve a stale dashboard. Note flakes copy only git-tracked files + # into the store, so committing the .json is the *only* way to also read it from disk; + # there is no "gitignore it and read it anyway" option. + fromGenerator = { name, generator }: + runCommand "grafana-dashboard-${name}.json" + { + nativeBuildInputs = [ jq python3 ]; + # panels.py sits next to the generators and is imported by both of them. + PYTHONPATH = generators; + # The store path is read-only; without this Python warns about __pycache__. + PYTHONDONTWRITEBYTECODE = 1; + } + '' + python3 ${generators}/${generator} | ${jqFixup} > "$out" + ''; in { # https://grafana.com/grafana/dashboards/1860 @@ -65,11 +98,10 @@ in # can never render against Prometheus, however it is provisioned. This is a # Prometheus-native replacement built on the metrics we actually export: node # exporter's zfs collector plus the ZFS textfile collector. - # Generated by dashboards/gen_zfs.py. - zfs-pools = fixup "zfs-pools" ../dashboards/zfs-pools.json; + zfs-pools = fromGenerator { name = "zfs-pools"; generator = "gen_zfs.py"; }; - # CrystalDiskInfo-style per-drive SMART view. Generated by dashboards/gen_smart.py. - # The Good/Caution/Bad verdict and the per-drive row titles both come from the - # recording rules in rules/smartctl.yml, so this one needs those loaded to be useful. - drive-health = fixup "drive-health" ../dashboards/smart.json; + # CrystalDiskInfo-style per-drive SMART view. The Good/Caution/Bad verdict and the + # per-drive row titles both come from the recording rules in rules/smartctl.yml, so + # this one needs those loaded to be useful. + drive-health = fromGenerator { name = "drive-health"; generator = "gen_smart.py"; }; } |