{ "title": "ZFS Pools", "uid": "zfs-pools", "description": "ZFS capacity, health, integrity, snapshot retention, dataset properties and performance. Fed by node_exporter's zfs and systemd collectors plus collectors/zfs-metrics.py from lgtm.nix.", "tags": [ "zfs", "storage" ], "timezone": "browser", "schemaVersion": 41, "version": 1, "editable": true, "graphTooltip": 1, "refresh": "1m", "time": { "from": "now-24h", "to": "now" }, "templating": { "list": [ { "name": "pool", "label": "Pool", "type": "query", "datasource": { "type": "prometheus", "uid": "prometheus" }, "query": { "qryType": 1, "query": "label_values(zfs_pool_health_code, pool)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "includeAll": true, "allValue": ".*", "multi": true, "current": { "text": [ "All" ], "value": [ "$__all" ] }, "options": [], "sort": 1 } ] }, "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "panels": [ { "type": "row", "title": "Pool Overview", "id": 1, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false, "panels": [] }, { "type": "stat", "title": "Pool Health", "description": "Health as reported by `zpool list -o health`.", "id": 2, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "type": "value", "options": { "0": { "text": "ONLINE", "color": "green", "index": 0 }, "1": { "text": "DEGRADED", "color": "orange", "index": 1 }, "2": { "text": "FAULTED", "color": "red", "index": 2 }, "3": { "text": "OFFLINE", "color": "text", "index": 3 }, "4": { "text": "UNAVAIL", "color": "red", "index": 4 }, "5": { "text": "REMOVED", "color": "purple", "index": 5 }, "6": { "text": "SUSPENDED", "color": "red", "index": 6 }, "7": { "text": "AVAIL", "color": "blue", "index": 7 }, "8": { "text": "UNKNOWN", "color": "text", "index": 8 } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_health_code{pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, "legendFormat": "{{pool}}" } ] }, { "type": "stat", "title": "Data Errors", "description": "Permanent, unrepairable errors from `zpool status`. Anything above zero means data has been lost and the affected files need restoring.", "id": 3, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(zfs_pool_data_errors{pool=~\"$pool\"}) or vector(0)", "refId": "A", "instant": true, "range": false, "legendFormat": "errors" } ] }, { "type": "stat", "title": "vdev Errors", "description": "Total read + write + checksum errors across all vdevs. Cleared by `zpool clear`.", "id": 4, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(zfs_pool_vdev_errors_total{pool=~\"$pool\"}) or vector(0)", "refId": "A", "instant": true, "range": false, "legendFormat": "errors" } ] }, { "type": "stat", "title": "Last Scrub", "description": "Time since the most recent completed scrub. Turns amber past ~5 weeks, which is a monthly scrub schedule having missed a run.", "id": 5, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 3024000 }, { "color": "red", "value": 6048000 } ] }, "unit": "s" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "time() - max(zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, "legendFormat": "ago" } ] }, { "type": "stat", "title": "Scrub / Resilver", "description": "", "id": 6, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "type": "value", "options": { "0": { "text": "idle", "color": "green", "index": 0 }, "1": { "text": "RUNNING", "color": "blue", "index": 1 } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max(zfs_pool_scrub_in_progress{pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, "legendFormat": "scrub" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max(zfs_pool_resilver_in_progress{pool=~\"$pool\"})", "refId": "B", "instant": true, "range": false, "legendFormat": "resilver" } ] }, { "type": "stat", "title": "Locked Datasets", "description": "Encrypted datasets whose key is not loaded. These are unreadable and are silently skipped by anything that walks the filesystem, including backups.", "id": 7, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "count(zfs_dataset_encrypted{pool=~\"$pool\"} == 1 and zfs_dataset_key_available{pool=~\"$pool\"} == 0) or vector(0)", "refId": "A", "instant": true, "range": false, "legendFormat": "locked" } ] }, { "type": "gauge", "title": "Space Used", "description": "How full the pool is: allocated / total. ZFS write performance degrades sharply past ~80%, and the pool becomes effectively read-only near 100%. The byte figures behind this are in 'Capacity by Pool' below.", "id": 8, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 5, "x": 0, "y": 5 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "max": 1, "unit": "percentunit", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.8 }, { "color": "red", "value": 0.9 } ] } }, "overrides": [] }, "options": { "showThresholdLabels": false, "showThresholdMarkers": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_capacity_ratio{pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, "legendFormat": "{{pool}}" } ] }, { "type": "gauge", "title": "Fragmentation", "description": "Free-space fragmentation.", "id": 9, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 5, "x": 5, "y": 5 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "max": 1, "unit": "percentunit", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "red", "value": 0.75 } ] } }, "overrides": [] }, "options": { "showThresholdLabels": false, "showThresholdMarkers": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, "legendFormat": "{{pool}}" } ] }, { "type": "stat", "title": "Free Space", "description": "", "id": 10, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 5, "x": 10, "y": 5 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, "legendFormat": "{{pool}}" } ] }, { "type": "stat", "title": "Snapshots Overdue", "description": "Dataset/label pairs whose newest snapshot is more than two intervals old \u2014 i.e. the auto-snapshot timer has missed at least one run.", "id": 11, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 4, "x": 15, "y": 5 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "count((time() - zfs_snapshot_newest_timestamp_seconds{pool=~\"$pool\"}) > on(label) group_left() (2 * zfs_snapshot_interval_seconds)) or vector(0)", "refId": "A", "instant": true, "range": false, "legendFormat": "overdue" } ] }, { "type": "stat", "title": "Retention Depth", "description": "Age of the oldest snapshot on the pool \u2014 how far back you can actually restore from.", "id": 12, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 5, "x": 19, "y": 5 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "time() - min(zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, "legendFormat": "oldest" } ] }, { "type": "timeseries", "title": "Pool Space Used / Free", "description": "Stacked, so the top of each pool's band is its total size and the split between the bands is the usage fraction the gauge above shows.", "id": 13, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 40, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_allocated_bytes{pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, "legendFormat": "{{pool}} used" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", "refId": "B", "instant": false, "range": true, "legendFormat": "{{pool}} free" } ] }, { "type": "timeseries", "title": "Fragmentation", "description": "Free-space fragmentation \u2014 how scattered the pool's remaining free space is, not how scattered the data is. It rises as a pool fills and stays high afterwards; only rewriting data (send/recv, or deleting a lot) brings it down. Kept on its own axis because it is unrelated to space used, and plotting the two together invites reading one as the other.", "id": 14, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_fragmentation_ratio{pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, "legendFormat": "{{pool}}" } ] }, { "type": "table", "title": "Capacity by Pool", "description": "Per-pool space in bytes. 'Size' is the raw pool size zpool reports, which for raidz is before parity overhead \u2014 so 'Free' is the number to trust for how much you can still write. 'Snapshots' is space held only by snapshots.", "id": 15, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false, "filterable": true }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "^(Size|Used|Free|Snapshots)$" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Used %" }, "properties": [ { "id": "unit", "value": "percentunit" }, { "id": "max", "value": 1 }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.8 }, { "color": "red", "value": 0.9 } ] } }, { "id": "custom.cellOptions", "value": { "type": "gauge", "mode": "gradient" } } ] }, { "matcher": { "id": "byName", "options": "Fragmentation" }, "properties": [ { "id": "unit", "value": "percentunit" } ] } ] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (pool) (zfs_pool_allocated_bytes{pool=~\"$pool\"})", "refId": "B", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (pool) (zfs_pool_free_bytes{pool=~\"$pool\"})", "refId": "C", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (pool) (zfs_pool_capacity_ratio{pool=~\"$pool\"})", "refId": "D", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (pool) (zfs_pool_fragmentation_ratio{pool=~\"$pool\"})", "refId": "E", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (pool) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})", "refId": "F", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (pool) (zfs_pool_dedup_ratio{pool=~\"$pool\"})", "refId": "G", "instant": true, "range": false, "format": "table" } ], "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "pattern": "pool|Value.*" } } }, { "id": "joinByField", "options": { "byField": "pool", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "pool": 0, "Value #A": 1, "Value #B": 2, "Value #C": 3, "Value #D": 4, "Value #E": 5, "Value #F": 6, "Value #G": 7 }, "renameByName": { "pool": "Pool", "Value #A": "Size", "Value #B": "Used", "Value #C": "Free", "Value #D": "Used %", "Value #E": "Fragmentation", "Value #F": "Snapshots", "Value #G": "Dedup" } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "Used %", "desc": true } ] } } ] }, { "type": "row", "title": "Integrity & Errors", "id": 16, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, "collapsed": false, "panels": [] }, { "type": "table", "title": "vdev State", "description": "Every vdev and leaf device in the selected pool(s).", "id": 17, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 26 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "type": "value", "options": { "0": { "text": "ONLINE", "color": "green", "index": 0 }, "1": { "text": "DEGRADED", "color": "orange", "index": 1 }, "2": { "text": "FAULTED", "color": "red", "index": 2 }, "3": { "text": "OFFLINE", "color": "text", "index": 3 }, "4": { "text": "UNAVAIL", "color": "red", "index": 4 }, "5": { "text": "REMOVED", "color": "purple", "index": 5 }, "6": { "text": "SUSPENDED", "color": "red", "index": 6 }, "7": { "text": "AVAIL", "color": "blue", "index": 7 }, "8": { "text": "UNKNOWN", "color": "text", "index": 8 } } } ], "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false, "filterable": true }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "State" }, "properties": [ { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } } ] } ] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_vdev_state_code{pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, "format": "table" } ], "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "pattern": "pool|vdev|Value" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "pool": "Pool", "vdev": "vdev", "Value": "State" } } } ] }, { "type": "timeseries", "title": "vdev Error Counters", "description": "Cumulative per-vdev errors. A counter that climbs is a disk on its way out.", "id": 18, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 26 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 0, "showPoints": "auto", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_vdev_errors_total{pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, "legendFormat": "{{pool}}/{{vdev}} {{type}}" } ] }, { "type": "table", "title": "Scrub & Resilver History", "description": "Result of the last completed scrub per pool. Errors here are blocks that could not be repaired from redundancy.", "id": 19, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false, "filterable": true }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Scrub Finished" }, "properties": [ { "id": "unit", "value": "dateTimeAsIso" } ] }, { "matcher": { "id": "byName", "options": "Resilver Finished" }, "properties": [ { "id": "unit", "value": "dateTimeAsIso" } ] }, { "matcher": { "id": "byName", "options": "Duration" }, "properties": [ { "id": "unit", "value": "s" } ] }, { "matcher": { "id": "byName", "options": "Repaired" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Errors" }, "properties": [ { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } } ] } ] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_last_scrub_end_timestamp_seconds{pool=~\"$pool\"} * 1000", "refId": "A", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_last_scrub_duration_seconds{pool=~\"$pool\"}", "refId": "B", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_last_scrub_errors{pool=~\"$pool\"}", "refId": "C", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_last_scrub_repaired_bytes{pool=~\"$pool\"}", "refId": "D", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_last_resilver_end_timestamp_seconds{pool=~\"$pool\"} * 1000", "refId": "E", "instant": true, "range": false, "format": "table" } ], "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "pattern": "pool|Value.*" } } }, { "id": "joinByField", "options": { "byField": "pool", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "pool": "Pool", "Value #A": "Scrub Finished", "Value #B": "Duration", "Value #C": "Errors", "Value #D": "Repaired", "Value #E": "Resilver Finished" } } } ] }, { "type": "timeseries", "title": "Pool Data Errors", "description": "Permanent data errors over time.", "id": 20, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_data_errors{pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, "legendFormat": "{{pool}}" } ] }, { "type": "row", "title": "Snapshots & Retention", "id": 21, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }, "collapsed": false, "panels": [] }, { "type": "bargauge", "title": "Time Since Last Snapshot", "description": "Age of the newest snapshot of each label. Compare against the label's own cadence \u2014 'frequent' is every 15 minutes, 'monthly' every month, so the colour thresholds here are only a coarse hint.", "id": 22, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 43 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 7200 }, { "color": "red", "value": 172800 } ] }, "unit": "s" }, "overrides": [] }, "options": { "displayMode": "basic", "orientation": "horizontal", "showUnfilled": true, "valueMode": "color", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "time() - max by (label) (zfs_pool_snapshot_newest_timestamp_seconds{pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, "legendFormat": "{{label}}" } ] }, { "type": "bargauge", "title": "Snapshots vs Retention Target", "description": "Worst dataset's snapshot count as a fraction of the configured services.zfs.autoSnapshot retention. 100% means the full set is on disk; a freshly-created dataset legitimately sits below target until it fills up.", "id": 23, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 43 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.9 } ] }, "unit": "percentunit", "max": 1.2 }, "overrides": [] }, "options": { "displayMode": "basic", "orientation": "horizontal", "showUnfilled": true, "valueMode": "color", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "min by (label) (zfs_snapshot_count{pool=~\"$pool\"} / on(label) group_left() zfs_snapshot_retention_target)", "refId": "A", "instant": true, "range": false, "legendFormat": "{{label}}" } ] }, { "type": "bargauge", "title": "Oldest Snapshot", "description": "How far back each label reaches.", "id": 24, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 43 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "options": { "displayMode": "basic", "orientation": "horizontal", "showUnfilled": true, "valueMode": "color", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "time() - min by (label) (zfs_pool_snapshot_oldest_timestamp_seconds{pool=~\"$pool\"})", "refId": "A", "instant": true, "range": false, "legendFormat": "{{label}}" } ] }, { "type": "timeseries", "title": "Snapshot Count by Label", "description": "Total snapshots per label. A steady sawtooth is healthy rotation; a monotonic decline means snapshots are expiring but not being replaced.", "id": 25, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (label) (zfs_snapshot_count{pool=~\"$pool\"})", "refId": "A", "instant": false, "range": true, "legendFormat": "{{label}}" } ] }, { "type": "timeseries", "title": "Snapshot Space by Label", "description": "Space held down by snapshots \u2014 the cost of your retention policy.", "id": 26, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 40, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (label) (zfs_pool_snapshot_used_bytes{pool=~\"$pool\"})", "refId": "A", "instant": false, "range": true, "legendFormat": "{{label}}" } ] }, { "type": "stat", "title": "Snapshot / Scrub Job Failures", "description": "systemd units for the auto-snapshot and auto-scrub timers currently in a failed state.", "id": 27, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 59 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "textMode": "auto", "wideLayout": true, "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(node_systemd_unit_state{name=~\"zfs-(snapshot|scrub).*\",state=\"failed\"}) or vector(0)", "refId": "A", "instant": true, "range": false, "legendFormat": "failed units" } ] }, { "type": "bargauge", "title": "Time Since Timer Last Fired", "description": "When each zfs-* systemd timer last triggered. This is the job side of the picture; the snapshot ages above are the result side.", "id": 28, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 6, "w": 18, "x": 6, "y": 59 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "options": { "displayMode": "basic", "orientation": "horizontal", "showUnfilled": true, "valueMode": "color", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "time() - node_systemd_timer_last_trigger_seconds{name=~\"zfs-.*\"}", "refId": "A", "instant": true, "range": false, "legendFormat": "{{name}}" } ] }, { "type": "row", "title": "Datasets", "id": 29, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 65 }, "collapsed": false, "panels": [] }, { "type": "timeseries", "title": "Dataset Usage Over Time", "description": "Space each dataset is responsible for, including its snapshots and descendants \u2014 so a parent's line contains its children's. Unstacked and unfilled so a single dataset's growth is readable against the others rather than riding on top of them. The legend carries last and max per dataset; the instant breakdown is in 'Capacity by Dataset' below.", "id": 30, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 66 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 0, "showPoints": "auto", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, "legendFormat": "{{dataset}}" } ] }, { "type": "table", "title": "Capacity by Dataset", "description": "Per-dataset space, separate from the per-pool figures above. 'Used' is the total a dataset is responsible for; 'Own Data' and 'Snapshots' are the two halves of that. 'Logical' is the size before compression, so Logical / Referenced is where the compression ratio comes from. 'Available' is shared with every sibling dataset unless a quota says otherwise.", "id": 31, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 75 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false, "filterable": true }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "^(Used|Own Data|Snapshots|Referenced|Logical|Available|Quota)$" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Quota" }, "properties": [ { "id": "noValue", "value": "none" } ] }, { "matcher": { "id": "byName", "options": "Ratio" }, "properties": [ { "id": "decimals", "value": 2 } ] } ] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_dataset_used_bytes{pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (dataset) (zfs_dataset_used_by_dataset_bytes{pool=~\"$pool\"})", "refId": "B", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (dataset) (zfs_dataset_used_by_snapshots_bytes{pool=~\"$pool\"})", "refId": "C", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (dataset) (zfs_dataset_referenced_bytes{pool=~\"$pool\"})", "refId": "D", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (dataset) (zfs_dataset_logical_used_bytes{pool=~\"$pool\"})", "refId": "E", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (dataset) (zfs_dataset_available_bytes{pool=~\"$pool\"})", "refId": "F", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (dataset) (zfs_dataset_compression_ratio{pool=~\"$pool\"})", "refId": "G", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "max by (dataset) (zfs_dataset_quota_bytes{pool=~\"$pool\"} > 0)", "refId": "H", "instant": true, "range": false, "format": "table" } ], "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "pattern": "pool|dataset|Value.*" } } }, { "id": "joinByField", "options": { "byField": "dataset", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "pool": 0, "dataset": 1, "Value #A": 2, "Value #B": 3, "Value #C": 4, "Value #D": 5, "Value #E": 6, "Value #F": 7, "Value #G": 8, "Value #H": 9 }, "renameByName": { "dataset": "Dataset", "pool": "Pool", "Value #A": "Used", "Value #B": "Own Data", "Value #C": "Snapshots", "Value #D": "Referenced", "Value #E": "Logical", "Value #F": "Available", "Value #G": "Ratio", "Value #H": "Quota" } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "Used", "desc": true } ] } } ] }, { "type": "table", "title": "Compression", "description": "group_left() attaches the algorithm label from the info metric onto the ratio, so this needs only one query and no join transformation.", "id": 32, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 83 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false, "filterable": true }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_dataset_compression_ratio{pool=~\"$pool\"} * on(pool, dataset) group_left(compression) zfs_dataset_compression{pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, "format": "table" } ], "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "pattern": "pool|dataset|compression|Value" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "pool": 0, "dataset": 1, "compression": 2, "Value": 3 }, "renameByName": { "pool": "Pool", "dataset": "Dataset", "compression": "Algorithm", "Value": "Ratio" } } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "Ratio", "desc": true } ] } } ] }, { "type": "table", "title": "Encryption", "description": "Encryption suite per dataset and the state of its key. Datasets that are not encrypted show suite 'off' and a dash \u2014 only an encrypted dataset can be locked.", "id": 33, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 83 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "type": "value", "options": { "0": { "text": "\u2014", "color": "text", "index": 0 }, "1": { "text": "unlocked", "color": "green", "index": 1 }, "2": { "text": "LOCKED", "color": "red", "index": 2 }, "3": { "text": "unknown", "color": "orange", "index": 3 } } } ], "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false, "filterable": true }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Key" }, "properties": [ { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } } ] } ] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_dataset_keystatus_code{pool=~\"$pool\"} * on(pool, dataset) group_left(encryption) zfs_dataset_encryption{pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, "format": "table" } ], "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "pattern": "pool|dataset|encryption|Value" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "pool": 0, "dataset": 1, "encryption": 2, "Value": 3 }, "renameByName": { "pool": "Pool", "dataset": "Dataset", "encryption": "Suite", "Value": "Key" } } } ] }, { "type": "row", "title": "Performance", "id": 34, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 91 }, "collapsed": false, "panels": [] }, { "type": "timeseries", "title": "Dataset Throughput", "description": "Per-dataset throughput from node_exporter's objset kstats. Writes are drawn negative.", "id": 35, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 92 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "rate(node_zfs_zpool_dataset_nread{zpool=~\"$pool\"}[$__rate_interval])", "refId": "A", "instant": false, "range": true, "legendFormat": "{{zpool}}/{{dataset}} read" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "-rate(node_zfs_zpool_dataset_nwritten{zpool=~\"$pool\"}[$__rate_interval])", "refId": "B", "instant": false, "range": true, "legendFormat": "{{zpool}}/{{dataset}} write" } ] }, { "type": "timeseries", "title": "Dataset IOPS", "description": "Writes are drawn negative.", "id": 36, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 92 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "iops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "rate(node_zfs_zpool_dataset_reads{zpool=~\"$pool\"}[$__rate_interval])", "refId": "A", "instant": false, "range": true, "legendFormat": "{{zpool}}/{{dataset}} read" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "-rate(node_zfs_zpool_dataset_writes{zpool=~\"$pool\"}[$__rate_interval])", "refId": "B", "instant": false, "range": true, "legendFormat": "{{zpool}}/{{dataset}} write" } ] }, { "type": "timeseries", "title": "ARC Size", "description": "", "id": 37, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 100 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "node_zfs_arc_size", "refId": "A", "instant": false, "range": true, "legendFormat": "size" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "node_zfs_arc_c_max", "refId": "B", "instant": false, "range": true, "legendFormat": "target max (c_max)" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "node_zfs_arc_c_min", "refId": "C", "instant": false, "range": true, "legendFormat": "target min (c_min)" } ] }, { "type": "timeseries", "title": "ARC Hit Ratio", "description": "clamp_min avoids a divide-by-zero while the pool is idle.", "id": 38, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 100 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "rate(node_zfs_arc_hits[$__rate_interval]) / clamp_min(rate(node_zfs_arc_hits[$__rate_interval]) + rate(node_zfs_arc_misses[$__rate_interval]), 1)", "refId": "A", "instant": false, "range": true, "legendFormat": "hit ratio" } ] }, { "type": "timeseries", "title": "ARC Hits / Misses", "description": "", "id": 39, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 100 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "rate(node_zfs_arc_hits[$__rate_interval])", "refId": "A", "instant": false, "range": true, "legendFormat": "hits" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "rate(node_zfs_arc_misses[$__rate_interval])", "refId": "B", "instant": false, "range": true, "legendFormat": "misses" } ] }, { "type": "row", "title": "Pool Internals", "id": 44, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 108 }, "collapsed": true, "panels": [ { "type": "timeseries", "title": "Reclaim & Leaks", "description": "`freeing` is space still being reclaimed in the background after a delete. `leaked` should always be zero.", "id": 40, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 109 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_freeing_bytes{pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, "legendFormat": "{{pool}} freeing" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_leaked_bytes{pool=~\"$pool\"}", "refId": "B", "instant": false, "range": true, "legendFormat": "{{pool}} leaked" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_checkpoint_bytes{pool=~\"$pool\"}", "refId": "C", "instant": false, "range": true, "legendFormat": "{{pool}} checkpoint" } ] }, { "type": "timeseries", "title": "Dedup Ratio", "description": "1.00 unless deduplication is enabled, which it usually should not be.", "id": 41, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 109 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_dedup_ratio{pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, "legendFormat": "{{pool}}" } ] }, { "type": "timeseries", "title": "Pool Health History", "description": "Health code over time \u2014 useful for spotting a pool that briefly degraded and recovered. zfs_pool_scan_state is also collected if you want to plot scrub/resilver phases alongside it.", "id": 42, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 109 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "stepAfter", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [ { "type": "value", "options": { "0": { "text": "ONLINE", "color": "green", "index": 0 }, "1": { "text": "DEGRADED", "color": "orange", "index": 1 }, "2": { "text": "FAULTED", "color": "red", "index": 2 }, "3": { "text": "OFFLINE", "color": "text", "index": 3 }, "4": { "text": "UNAVAIL", "color": "red", "index": 4 }, "5": { "text": "REMOVED", "color": "purple", "index": 5 }, "6": { "text": "SUSPENDED", "color": "red", "index": 6 }, "7": { "text": "AVAIL", "color": "blue", "index": 7 }, "8": { "text": "UNKNOWN", "color": "text", "index": 8 } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "min": 0, "max": 8 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_pool_health_code{pool=~\"$pool\"}", "refId": "A", "instant": false, "range": true, "legendFormat": "{{pool}}" } ] }, { "type": "table", "title": "Dataset Properties", "description": "A quota of 0 means unlimited. Auto-snapshot reflects the com.sun:auto-snapshot property that zfs-auto-snapshot honours.", "id": 43, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 116 }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false, "filterable": true }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Record Size" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Quota" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Ref Quota" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Auto-snapshot" }, "properties": [ { "id": "mappings", "value": [ { "type": "value", "options": { "0": { "text": "no", "color": "orange", "index": 0 }, "1": { "text": "yes", "color": "green", "index": 1 } } } ] } ] } ] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_dataset_recordsize_bytes{pool=~\"$pool\"}", "refId": "A", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_dataset_quota_bytes{pool=~\"$pool\"}", "refId": "B", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_dataset_refquota_bytes{pool=~\"$pool\"}", "refId": "C", "instant": true, "range": false, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_dataset_autosnapshot_enabled{pool=~\"$pool\"}", "refId": "D", "instant": true, "range": false, "format": "table" } ], "transformations": [ { "id": "filterFieldsByName", "options": { "include": { "pattern": "dataset|Value.*" } } }, { "id": "joinByField", "options": { "byField": "dataset", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "dataset": "Dataset", "Value #A": "Record Size", "Value #B": "Quota", "Value #C": "Ref Quota", "Value #D": "Auto-snapshot" } } } ] } ] }, { "type": "row", "title": "Collector Health", "id": 48, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 109 }, "collapsed": true, "panels": [ { "type": "timeseries", "title": "ZFS Collector Duration", "description": "How long collectors/zfs-metrics.py takes. Listing every snapshot is the slow part; if this approaches the 1 minute timer interval, raise OnUnitActiveSec.", "id": 45, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 110 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "zfs_metrics_collection_duration_seconds", "refId": "A", "instant": false, "range": true, "legendFormat": "collection" } ] }, { "type": "timeseries", "title": "Textfile Age", "description": "Age of each .prom file. If this climbs without bound the collector has stopped writing and every ZFS metric above is stale.", "id": 46, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 110 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s", "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "time() - node_textfile_mtime_seconds", "refId": "A", "instant": false, "range": true, "legendFormat": "{{file}}" } ] }, { "type": "timeseries", "title": "Scrape Health", "description": "node_textfile_scrape_error goes to 1 if a .prom file is malformed, in which case node_exporter drops it entirely.", "id": 47, "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 110 }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 15, "showPoints": "never", "spanNulls": true, "axisBorderShow": false, "axisCenteredZero": false, "gradientMode": "none", "pointSize": 5, "scaleDistribution": { "type": "linear" }, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "up{job=\"node\"}", "refId": "A", "instant": false, "range": true, "legendFormat": "node exporter up" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "node_textfile_scrape_error", "refId": "B", "instant": false, "range": true, "legendFormat": "textfile parse error" } ] } ] } ] }