2020-03-27 12:35:57 +01:00
|
|
|
package zfscmd
|
|
|
|
|
|
|
|
import (
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
)
|
|
|
|
|
|
|
|
var metrics struct {
|
|
|
|
totaltime *prometheus.HistogramVec
|
|
|
|
systemtime *prometheus.HistogramVec
|
|
|
|
usertime *prometheus.HistogramVec
|
|
|
|
}
|
|
|
|
|
|
|
|
var timeLabels = []string{"jobid", "zfsbinary", "zfsverb"}
|
|
|
|
var timeBuckets = []float64{0.01, 0.1, 0.2, 0.5, 0.75, 1, 2, 5, 10, 60}
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
metrics.totaltime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
|
|
Namespace: "zrepl",
|
|
|
|
Subsystem: "zfscmd",
|
|
|
|
Name: "runtime",
|
|
|
|
Help: "number of seconds that the command took from start until wait returned",
|
|
|
|
Buckets: timeBuckets,
|
|
|
|
}, timeLabels)
|
|
|
|
metrics.systemtime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
|
|
Namespace: "zrepl",
|
|
|
|
Subsystem: "zfscmd",
|
|
|
|
Name: "systemtime",
|
|
|
|
Help: "https://golang.org/pkg/os/#ProcessState.SystemTime",
|
|
|
|
Buckets: timeBuckets,
|
|
|
|
}, timeLabels)
|
|
|
|
metrics.usertime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
|
|
Namespace: "zrepl",
|
|
|
|
Subsystem: "zfscmd",
|
|
|
|
Name: "usertime",
|
|
|
|
Help: "https://golang.org/pkg/os/#ProcessState.UserTime",
|
|
|
|
Buckets: timeBuckets,
|
|
|
|
}, timeLabels)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func RegisterMetrics(r prometheus.Registerer) {
|
|
|
|
r.MustRegister(metrics.totaltime)
|
|
|
|
r.MustRegister(metrics.systemtime)
|
|
|
|
r.MustRegister(metrics.usertime)
|
|
|
|
}
|
|
|
|
|
zfscmd: fix crash in zfscmd_prometheus.go due to incorrectly extracted ProcessState
fixup of 96e188d7c4837ff1188ab923daeb0a0f4cce31ae
refs #196
refs #301
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x10 pc=0x9a472a]
goroutine 15826 [running]:
os.(*ProcessState).systemTime(...)
/home/cs/go1.13/src/os/exec_unix.go:98
os.(*ProcessState).SystemTime(...)
/home/cs/go1.13/src/os/exec.go:141
github.com/zrepl/zrepl/zfs/zfscmd.waitPostPrometheus(0xc000c04800, 0xe21ce0, 0xc000068270, 0xbf9f80d88107e861, 0x19bae710e6, 0x13a8b60)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd_prometheus.go:69 +0x22a
github.com/zrepl/zrepl/zfs/zfscmd.(*Cmd).waitPost(0xc000c04800, 0xe21ce0, 0xc000068270)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd.go:155 +0x18a
github.com/zrepl/zrepl/zfs/zfscmd.(*Cmd).CombinedOutput(0xc000c04800, 0xc0004b8270, 0xd02eea, 0x3, 0xc0001f6c40, 0x3)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd.go:40 +0xb3
github.com/zrepl/zrepl/zfs.ZFSRelease(0xe36aa0, 0xc0004b8270, 0xc0009a3a40, 0x13, 0xc0004a5d00, 0x1, 0x1, 0xed62eb221, 0x13a8b60)
/home/cs/zrepl/zrepl/zfs/holds.go:102 +0x2a7
github.com/zrepl/zrepl/endpoint.ReleaseStep(0xe36aa0, 0xc0004b8270, 0xc0004befc0, 0xe, 0xd08482, 0x8, 0xc0001cb02f, 0x2, 0x1eeea3bff89dc90b, 0x134d6, ...)
/home/cs/zrepl/zrepl/endpoint/endpoint_zfs_abstraction_step.go:130 +0x367
github.com/zrepl/zrepl/endpoint.(*Sender).SendCompleted.func2(0xc000459190, 0xc000390e30, 0xc00041fd80, 0xc0004befc0, 0xe, 0xd08482, 0x8, 0xc0001cb02f, 0x2, 0x1eeea3bff89dc90b, ...)
/home/cs/zrepl/zrepl/endpoint/endpoint.go:419 +0x1c3
created by github.com/zrepl/zrepl/endpoint.(*Sender).SendCompleted
/home/cs/zrepl/zrepl/endpoint/endpoint.go:413 +0x776
2020-04-20 13:57:19 +02:00
|
|
|
func waitPostPrometheus(c *Cmd, u usage, err error, now time.Time) {
|
2020-03-27 12:35:57 +01:00
|
|
|
|
|
|
|
if len(c.cmd.Args) < 2 {
|
|
|
|
getLogger(c.ctx).WithField("args", c.cmd.Args).
|
|
|
|
Warn("prometheus: cannot turn zfs command into metric")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Note: do not start parsing other aspects
|
|
|
|
// of the ZFS command line. This is not the suitable layer
|
|
|
|
// for such a task.
|
|
|
|
|
|
|
|
jobid := getJobIDOrDefault(c.ctx, "_nojobid")
|
|
|
|
|
|
|
|
labelValues := []string{jobid, c.cmd.Args[0], c.cmd.Args[1]}
|
|
|
|
|
|
|
|
metrics.totaltime.
|
|
|
|
WithLabelValues(labelValues...).
|
zfscmd: fix crash in zfscmd_prometheus.go due to incorrectly extracted ProcessState
fixup of 96e188d7c4837ff1188ab923daeb0a0f4cce31ae
refs #196
refs #301
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x10 pc=0x9a472a]
goroutine 15826 [running]:
os.(*ProcessState).systemTime(...)
/home/cs/go1.13/src/os/exec_unix.go:98
os.(*ProcessState).SystemTime(...)
/home/cs/go1.13/src/os/exec.go:141
github.com/zrepl/zrepl/zfs/zfscmd.waitPostPrometheus(0xc000c04800, 0xe21ce0, 0xc000068270, 0xbf9f80d88107e861, 0x19bae710e6, 0x13a8b60)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd_prometheus.go:69 +0x22a
github.com/zrepl/zrepl/zfs/zfscmd.(*Cmd).waitPost(0xc000c04800, 0xe21ce0, 0xc000068270)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd.go:155 +0x18a
github.com/zrepl/zrepl/zfs/zfscmd.(*Cmd).CombinedOutput(0xc000c04800, 0xc0004b8270, 0xd02eea, 0x3, 0xc0001f6c40, 0x3)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd.go:40 +0xb3
github.com/zrepl/zrepl/zfs.ZFSRelease(0xe36aa0, 0xc0004b8270, 0xc0009a3a40, 0x13, 0xc0004a5d00, 0x1, 0x1, 0xed62eb221, 0x13a8b60)
/home/cs/zrepl/zrepl/zfs/holds.go:102 +0x2a7
github.com/zrepl/zrepl/endpoint.ReleaseStep(0xe36aa0, 0xc0004b8270, 0xc0004befc0, 0xe, 0xd08482, 0x8, 0xc0001cb02f, 0x2, 0x1eeea3bff89dc90b, 0x134d6, ...)
/home/cs/zrepl/zrepl/endpoint/endpoint_zfs_abstraction_step.go:130 +0x367
github.com/zrepl/zrepl/endpoint.(*Sender).SendCompleted.func2(0xc000459190, 0xc000390e30, 0xc00041fd80, 0xc0004befc0, 0xe, 0xd08482, 0x8, 0xc0001cb02f, 0x2, 0x1eeea3bff89dc90b, ...)
/home/cs/zrepl/zrepl/endpoint/endpoint.go:419 +0x1c3
created by github.com/zrepl/zrepl/endpoint.(*Sender).SendCompleted
/home/cs/zrepl/zrepl/endpoint/endpoint.go:413 +0x776
2020-04-20 13:57:19 +02:00
|
|
|
Observe(u.total_secs)
|
2020-03-27 12:35:57 +01:00
|
|
|
metrics.systemtime.WithLabelValues(labelValues...).
|
zfscmd: fix crash in zfscmd_prometheus.go due to incorrectly extracted ProcessState
fixup of 96e188d7c4837ff1188ab923daeb0a0f4cce31ae
refs #196
refs #301
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x10 pc=0x9a472a]
goroutine 15826 [running]:
os.(*ProcessState).systemTime(...)
/home/cs/go1.13/src/os/exec_unix.go:98
os.(*ProcessState).SystemTime(...)
/home/cs/go1.13/src/os/exec.go:141
github.com/zrepl/zrepl/zfs/zfscmd.waitPostPrometheus(0xc000c04800, 0xe21ce0, 0xc000068270, 0xbf9f80d88107e861, 0x19bae710e6, 0x13a8b60)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd_prometheus.go:69 +0x22a
github.com/zrepl/zrepl/zfs/zfscmd.(*Cmd).waitPost(0xc000c04800, 0xe21ce0, 0xc000068270)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd.go:155 +0x18a
github.com/zrepl/zrepl/zfs/zfscmd.(*Cmd).CombinedOutput(0xc000c04800, 0xc0004b8270, 0xd02eea, 0x3, 0xc0001f6c40, 0x3)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd.go:40 +0xb3
github.com/zrepl/zrepl/zfs.ZFSRelease(0xe36aa0, 0xc0004b8270, 0xc0009a3a40, 0x13, 0xc0004a5d00, 0x1, 0x1, 0xed62eb221, 0x13a8b60)
/home/cs/zrepl/zrepl/zfs/holds.go:102 +0x2a7
github.com/zrepl/zrepl/endpoint.ReleaseStep(0xe36aa0, 0xc0004b8270, 0xc0004befc0, 0xe, 0xd08482, 0x8, 0xc0001cb02f, 0x2, 0x1eeea3bff89dc90b, 0x134d6, ...)
/home/cs/zrepl/zrepl/endpoint/endpoint_zfs_abstraction_step.go:130 +0x367
github.com/zrepl/zrepl/endpoint.(*Sender).SendCompleted.func2(0xc000459190, 0xc000390e30, 0xc00041fd80, 0xc0004befc0, 0xe, 0xd08482, 0x8, 0xc0001cb02f, 0x2, 0x1eeea3bff89dc90b, ...)
/home/cs/zrepl/zrepl/endpoint/endpoint.go:419 +0x1c3
created by github.com/zrepl/zrepl/endpoint.(*Sender).SendCompleted
/home/cs/zrepl/zrepl/endpoint/endpoint.go:413 +0x776
2020-04-20 13:57:19 +02:00
|
|
|
Observe(u.system_secs)
|
2020-03-27 12:35:57 +01:00
|
|
|
metrics.usertime.WithLabelValues(labelValues...).
|
zfscmd: fix crash in zfscmd_prometheus.go due to incorrectly extracted ProcessState
fixup of 96e188d7c4837ff1188ab923daeb0a0f4cce31ae
refs #196
refs #301
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x10 pc=0x9a472a]
goroutine 15826 [running]:
os.(*ProcessState).systemTime(...)
/home/cs/go1.13/src/os/exec_unix.go:98
os.(*ProcessState).SystemTime(...)
/home/cs/go1.13/src/os/exec.go:141
github.com/zrepl/zrepl/zfs/zfscmd.waitPostPrometheus(0xc000c04800, 0xe21ce0, 0xc000068270, 0xbf9f80d88107e861, 0x19bae710e6, 0x13a8b60)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd_prometheus.go:69 +0x22a
github.com/zrepl/zrepl/zfs/zfscmd.(*Cmd).waitPost(0xc000c04800, 0xe21ce0, 0xc000068270)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd.go:155 +0x18a
github.com/zrepl/zrepl/zfs/zfscmd.(*Cmd).CombinedOutput(0xc000c04800, 0xc0004b8270, 0xd02eea, 0x3, 0xc0001f6c40, 0x3)
/home/cs/zrepl/zrepl/zfs/zfscmd/zfscmd.go:40 +0xb3
github.com/zrepl/zrepl/zfs.ZFSRelease(0xe36aa0, 0xc0004b8270, 0xc0009a3a40, 0x13, 0xc0004a5d00, 0x1, 0x1, 0xed62eb221, 0x13a8b60)
/home/cs/zrepl/zrepl/zfs/holds.go:102 +0x2a7
github.com/zrepl/zrepl/endpoint.ReleaseStep(0xe36aa0, 0xc0004b8270, 0xc0004befc0, 0xe, 0xd08482, 0x8, 0xc0001cb02f, 0x2, 0x1eeea3bff89dc90b, 0x134d6, ...)
/home/cs/zrepl/zrepl/endpoint/endpoint_zfs_abstraction_step.go:130 +0x367
github.com/zrepl/zrepl/endpoint.(*Sender).SendCompleted.func2(0xc000459190, 0xc000390e30, 0xc00041fd80, 0xc0004befc0, 0xe, 0xd08482, 0x8, 0xc0001cb02f, 0x2, 0x1eeea3bff89dc90b, ...)
/home/cs/zrepl/zrepl/endpoint/endpoint.go:419 +0x1c3
created by github.com/zrepl/zrepl/endpoint.(*Sender).SendCompleted
/home/cs/zrepl/zrepl/endpoint/endpoint.go:413 +0x776
2020-04-20 13:57:19 +02:00
|
|
|
Observe(u.user_secs)
|
2020-03-27 12:35:57 +01:00
|
|
|
|
|
|
|
}
|