extern/zrepl
1
0
Fork 0
mirror of https://github.com/zrepl/zrepl.git synced 2024-11-21 16:03:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

metric to detect filesystems rules that don't match any local dataset (#653)

Browse Source 
This PR adds a Prometheus counter called
`zrepl_zfs_list_unmatched_user_specified_dataset_count`.
Monitor for increases of the counter to detect filesystem filter rules that
have no effect because they don't match any local filesystem.

An example use case for this is the following story:
1. Someone sets up zrepl with `filesystems` filter for `zroot/pg14<`.
2. During the upgrade to Postgres 15, they rename the dataset to `zroot/pg15`,
   but forget to update the zrepl `filesystems` filter.
3. zrepl will not snapshot / replicate the `zroot/pg15<` datasets.

Since `filesystems` rules are always evaluated on the side that has the datasets,
we can smuggle this functionality into the `zfs` module's `ZFSList` function that
is used by all jobs with a `filesystems` filter.

Dashboard changes:
- histogram with increase in $__interval, one row per job
- table with increase in $__range
- explainer text box, so, people know what the previous two are about
We had to re-arrange some panels, hence the Git diff isn't great.

closes https://github.com/zrepl/zrepl/pull/653

Co-authored-by: Christian Schwarz <me@cschwarz.com>
Co-authored-by: Goran Mekić <meka@tilda.center>
This commit is contained in:
Goran Mekic 2023-05-02 22:13:52 +02:00 committed by GitHub
parent 2b3daaf9f1
commit bc5e1ede04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1536 additions and 1257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
daemon/filters/fsmapfilter.go
View File

@ -160,6 +160,14 @@ func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
return
}
func (m DatasetMapFilter) UserSpecifiedDatasets() (datasets zfs.UserSpecifiedDatasetsSet) {
datasets = make(zfs.UserSpecifiedDatasetsSet)
for i := range m.entries {
datasets[m.entries[i].path.ToString()] = true
}
return
}
// Construct a new filter-only DatasetMapFilter from a mapping
// The new filter allows exactly those paths that were not forbidden by the mapping.
func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {

633
dist/grafana/grafana-prometheus-zrepl.json vendored
View File

@ -23,6 +23,12 @@
"name": "Graph (old)",
"version": ""
},
{
"type": "panel",
"id": "heatmap",
"name": "Heatmap",
"version": ""
},
{
"type": "datasource",
"id": "prometheus",
@ -35,6 +41,12 @@
"name": "Stat",
"version": ""
},
{
"type": "panel",
"id": "table",
"name": "Table",
"version": ""
},
{
"type": "panel",
"id": "text",
@ -89,7 +101,7 @@
"showLineNumbers": false,
"showMiniMap": false
},
"content": "# zrepl Prometheus Metrics\n\nzrepl exposes Prometheus metrics and ships with this Grafana dashboard.\nThe exported metrics are suitable for health checks:\n\n* The log should generally be warning & error-free\n * The `Log Messages that require attention` graph visualizes log message at levels that generally indicate problems.\n* The number of goroutines should not grow unboundedly over time.\n * During replication, the number of goroutines can be way higher than during idle time.\n * If the goroutine count grows with each replication, there is clearly a goroutine leak. Please open a bug report.\n* Memory consumption should not grow unboundedly over time.\n * Note that the Go runtime pre-allocates some of its heap from the OS.\n * zrepl actually uses much less memory than allocated from the OS.\n * Since Go 1.11, Go pre-allocates more aggressively.\n* Monitor that some data is replicated, although that metric does not guarantee that replication was successful.\n\n**In general, note that the exported metrics are not stable unless declared otherwise.**",
"content": "# zrepl Prometheus Metrics\n\nzrepl exposes Prometheus metrics and ships with this Grafana dashboard.\nThe exported metrics are suitable for health checks:\n\n* The log should generally be warning & error-free\n * The `Log Messages that require attention` graph visualizes log message at levels that generally indicate problems.\n* In most setups, there shouldn't be any unmatched filesystem filter rules.\n* The number of goroutines should not grow unboundedly over time.\n * During replication, the number of goroutines can be way higher than during idle time.\n * If the goroutine count grows with each replication, there is clearly a goroutine leak. Please open a bug report.\n* Memory consumption should not grow unboundedly over time.\n * Note that the Go runtime pre-allocates some of its heap from the OS.\n * zrepl actually uses much less memory than allocated from the OS.\n * Since Go 1.11, Go pre-allocates more aggressively.\n* Monitor that some data is replicated, although that metric does not guarantee that replication was successful.\n\n**In general, note that the exported metrics are not stable unless declared otherwise.**",
"mode": "markdown"
},
"pluginVersion": "9.3.6",
@ -414,6 +426,407 @@
"align": false
}
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 3,
"w": 5,
"x": 0,
"y": 17
},
"id": 56,
"options": {
"code": {
"language": "plaintext",
"showLineNumbers": false,
"showMiniMap": false
},
"content": "### Unmatched Filesystems Rules\n\nFilesystem filter rules which mention datasets that didn't exist in the `zfs list` output.",
"mode": "markdown"
},
"pluginVersion": "9.3.6",
"type": "text"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 7,
"x": 5,
"y": 17
},
"id": 52,
"maxDataPoints": 10,
"options": {
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-red",
"min": 0,
"mode": "opacity",
"reverse": false,
"scale": "linear",
"scheme": "Oranges",
"steps": 2
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": false
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"show": true,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"reverse": false
}
},
"pluginVersion": "9.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "increase(zrepl_zfs_list_unmatched_user_specified_dataset_count{job=\"$prom_job_name\"}[$__interval])",
"format": "time_series",
"legendFormat": "{{jobid}}",
"range": true,
"refId": "A"
}
],
"title": "Occurences increase[$__interval]",
"transformations": [],
"type": "heatmap"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 4,
"w": 12,
"x": 12,
"y": 18
},
"hiddenSeries": false,
"id": 22,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.6",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "increase(zrepl_daemon_log_entries{job='$prom_job_name',level=~'warn|error'}[1m])",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Log Messages that require attention",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"min": "0",
"show": true
},
{
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "color-background-solid",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "dark-red",
"value": 0.01
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "jobid"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "transparent",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 5,
"x": 0,
"y": 20
},
"id": 54,
"maxDataPoints": 20,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "Value"
}
]
},
"pluginVersion": "9.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"exemplar": false,
"expr": "increase(zrepl_zfs_list_unmatched_user_specified_dataset_count{job=\"$prom_job_name\"}[$__range])",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "{{jobid}}",
"range": false,
"refId": "A"
}
],
"title": "Occurences [Dashboard Range]",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Value": false,
"instance": true,
"job": true
},
"indexByName": {},
"renameByName": {}
}
}
],
"type": "table"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"links": []
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 5,
"w": 12,
"x": 12,
"y": 22
},
"hiddenSeries": false,
"id": 23,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.6",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(increase(zrepl_daemon_log_entries{job='$prom_job_name',zrepl_job=~\"^[^_].*\"}[1m])) by (instance,zrepl_job)",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Log Activity (without internal jobs)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"min": "0",
"show": true
},
{
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": true,
@ -435,7 +848,7 @@
"h": 5,
"w": 12,
"x": 0,
"y": 17
"y": 25
},
"hiddenSeries": false,
"id": 42,
@ -535,13 +948,13 @@
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 4,
"h": 5,
"w": 12,
"x": 12,
"y": 18
"y": 27
},
"hiddenSeries": false,
"id": 22,
"id": 47,
"legend": {
"avg": false,
"current": false,
@ -573,7 +986,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "increase(zrepl_daemon_log_entries{job='$prom_job_name',level=~'warn|error'}[1m])",
"expr": "zrepl_endpoint_abstractions_cache_entry_count",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
@ -581,7 +994,7 @@
],
"thresholds": [],
"timeRegions": [],
"title": "Log Messages that require attention",
"title": "zfs abstractions cache entry count (should not be zero and not grow unboundedly)",
"tooltip": {
"shared": true,
"sort": 0,
@ -597,7 +1010,6 @@
{
"format": "short",
"logBase": 1,
"min": "0",
"show": true
},
{
@ -631,7 +1043,7 @@
"h": 5,
"w": 12,
"x": 0,
"y": 22
"y": 30
},
"hiddenSeries": false,
"id": 33,
@ -740,104 +1152,11 @@
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 5,
"w": 12,
"x": 12,
"y": 22
},
"hiddenSeries": false,
"id": 23,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.6",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(increase(zrepl_daemon_log_entries{job='$prom_job_name',zrepl_job=~\"^[^_].*\"}[1m])) by (instance,zrepl_job)",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Log Activity (without internal jobs)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"min": "0",
"show": true
},
{
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 5,
"w": 12,
"x": 0,
"y": 27
"y": 35
},
"hiddenSeries": false,
"id": 41,
@ -856,9 +1175,10 @@
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.6",
"pointradius": 5,
"points": false,
"renderer": "flot",
@ -920,98 +1240,7 @@
},
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 5,
"w": 12,
"x": 12,
"y": 27
},
"hiddenSeries": false,
"id": 47,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "zrepl_endpoint_abstractions_cache_entry_count",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "zfs abstractions cache entry count (should not be zero and not grow unboundedly)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"show": true
},
{
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {}
"links": []
},
"overrides": []
},
@ -1021,7 +1250,7 @@
"h": 5,
"w": 12,
"x": 0,
"y": 32
"y": 40
},
"hiddenSeries": false,
"id": 17,
@ -1039,9 +1268,10 @@
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.6",
"pointradius": 5,
"points": false,
"renderer": "flot",
@ -1104,7 +1334,7 @@
},
"fieldConfig": {
"defaults": {
"custom": {}
"links": []
},
"overrides": []
},
@ -1114,7 +1344,7 @@
"h": 5,
"w": 12,
"x": 0,
"y": 37
"y": 45
},
"hiddenSeries": false,
"id": 19,
@ -1132,9 +1362,10 @@
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.6",
"pointradius": 5,
"points": false,
"renderer": "flot",

7
endpoint/endpoint.go
View File

@ -428,6 +428,7 @@ func (p *Sender) Receive(ctx context.Context, r *pdu.ReceiveReq, _ io.ReadCloser
type FSFilter interface { // FIXME unused
Filter(path *zfs.DatasetPath) (pass bool, err error)
UserSpecifiedDatasets() zfs.UserSpecifiedDatasetsSet
}
// FIXME: can we get away without error types here?
@ -587,6 +588,12 @@ func (f subroot) Filter(p *zfs.DatasetPath) (pass bool, err error) {
return p.HasPrefix(f.localRoot) && !p.Equal(f.localRoot), nil
}
func (f subroot) UserSpecifiedDatasets() zfs.UserSpecifiedDatasetsSet {
return zfs.UserSpecifiedDatasetsSet{
f.localRoot.ToString(): true,
}
}
func (f subroot) MapToLocal(fs string) (*zfs.DatasetPath, error) {
p, err := zfs.NewDatasetPath(fs)
if err != nil {

16
zfs/mapping.go
View File

@ -3,12 +3,20 @@ package zfs
import (
"context"
"fmt"
"github.com/zrepl/zrepl/zfs/zfscmd"
)
type DatasetFilter interface {
Filter(p *DatasetPath) (pass bool, err error)
// The caller owns the returned set.
// Implementations should return a copy.
UserSpecifiedDatasets() UserSpecifiedDatasetsSet
}
// A set of dataset names that the user specified in the configuration file.
type UserSpecifiedDatasetsSet map[string]bool
// Returns a DatasetFilter that does not filter (passes all paths)
func NoFilter() DatasetFilter {
return noFilter{}
@ -19,6 +27,7 @@ type noFilter struct{}
var _ DatasetFilter = noFilter{}
func (noFilter) Filter(p *DatasetPath) (pass bool, err error) { return true, nil }
func (noFilter) UserSpecifiedDatasets() UserSpecifiedDatasetsSet { return nil }
func ZFSListMapping(ctx context.Context, filter DatasetFilter) (datasets []*DatasetPath, err error) {
res, err := ZFSListMappingProperties(ctx, filter, nil)
@ -61,6 +70,7 @@ func ZFSListMappingProperties(ctx context.Context, filter DatasetFilter, propert
go ZFSListChan(ctx, rchan, properties, nil, "-r", "-t", "filesystem,volume")
unmatchedUserSpecifiedDatasets := filter.UserSpecifiedDatasets()
datasets = make([]ZFSListMappingPropertiesResult, 0)
for r := range rchan {
@ -74,6 +84,8 @@ func ZFSListMappingProperties(ctx context.Context, filter DatasetFilter, propert
return
}
delete(unmatchedUserSpecifiedDatasets, path.ToString())
pass, filterErr := filter.Filter(path)
if filterErr != nil {
return nil, fmt.Errorf("error calling filter: %s", filterErr)
@ -87,5 +99,9 @@ func ZFSListMappingProperties(ctx context.Context, filter DatasetFilter, propert
}
jobid := zfscmd.GetJobIDOrDefault(ctx, "__nojobid")
metric := prom.ZFSListUnmatchedUserSpecifiedDatasetCount.WithLabelValues(jobid)
metric.Add(float64(len(unmatchedUserSpecifiedDatasets)))
return
}

13
zfs/prometheus.go
View File

@ -7,6 +7,7 @@ var prom struct {
ZFSSnapshotDuration *prometheus.HistogramVec
ZFSBookmarkDuration *prometheus.HistogramVec
ZFSDestroyDuration *prometheus.HistogramVec
ZFSListUnmatchedUserSpecifiedDatasetCount *prometheus.GaugeVec
}
func init() {
@ -34,6 +35,15 @@ func init() {
Name: "destroy_duration",
Help: "Duration it took to destroy a dataset",
}, []string{"dataset_type", "filesystem"})
prom.ZFSListUnmatchedUserSpecifiedDatasetCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "zrepl",
Subsystem: "zfs",
Name: "list_unmatched_user_specified_dataset_count",
Help: "When evaluating a DatsetFilter against zfs list output, this counter " +
"is incremented for every DatasetFilter rule that did not match any " +
"filesystem name in the zfs list output. Monitor for increases to detect filesystem " +
"filter rules that have no effect because they don't match any local filesystem.",
}, []string{"jobid"})
}
func PrometheusRegister(registry prometheus.Registerer) error {
@ -49,5 +59,8 @@ func PrometheusRegister(registry prometheus.Registerer) error {
if err := registry.Register(prom.ZFSDestroyDuration); err != nil {
return err
}
if err := registry.Register(prom.ZFSListUnmatchedUserSpecifiedDatasetCount); err != nil {
return err
}
return nil
}

4
zfs/zfscmd/zfscmd_context.go
View File

@ -19,6 +19,10 @@ func WithJobID(ctx context.Context, jobID string) context.Context {
return context.WithValue(ctx, contextKeyJobID, jobID)
}
func GetJobIDOrDefault(ctx context.Context, def string) string {
return getJobIDOrDefault(ctx, def)
}
func getJobIDOrDefault(ctx context.Context, def string) string {
ret, ok := ctx.Value(contextKeyJobID).(string)
if !ok {