2018-08-27 19:10:55 +02:00
|
|
|
package daemon
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2019-03-22 19:41:12 +01:00
|
|
|
"net"
|
|
|
|
"net/http"
|
|
|
|
|
2018-08-27 19:10:55 +02:00
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
2019-03-22 19:41:12 +01:00
|
|
|
|
2018-09-08 07:03:41 +02:00
|
|
|
"github.com/zrepl/zrepl/config"
|
2018-08-27 22:21:45 +02:00
|
|
|
"github.com/zrepl/zrepl/daemon/job"
|
2018-09-08 07:03:41 +02:00
|
|
|
"github.com/zrepl/zrepl/logger"
|
2018-12-11 22:01:50 +01:00
|
|
|
"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
|
2018-08-27 19:10:55 +02:00
|
|
|
"github.com/zrepl/zrepl/zfs"
|
|
|
|
)
|
|
|
|
|
|
|
|
type prometheusJob struct {
|
|
|
|
listen string
|
|
|
|
}
|
|
|
|
|
2018-09-08 07:03:41 +02:00
|
|
|
func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
|
|
|
|
if _, _, err := net.SplitHostPort(in.Listen); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return &prometheusJob{in.Listen}, nil
|
2018-08-27 19:10:55 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
var prom struct {
|
2019-03-22 19:41:12 +01:00
|
|
|
taskLogEntries *prometheus.CounterVec
|
2018-08-27 19:10:55 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
prom.taskLogEntries = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
|
|
Namespace: "zrepl",
|
|
|
|
Subsystem: "daemon",
|
2018-09-08 07:03:41 +02:00
|
|
|
Name: "log_entries",
|
2018-08-27 19:10:55 +02:00
|
|
|
Help: "number of log entries per job task and level",
|
2018-09-08 07:03:41 +02:00
|
|
|
}, []string{"zrepl_job", "level"})
|
2018-08-27 19:10:55 +02:00
|
|
|
prometheus.MustRegister(prom.taskLogEntries)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (j *prometheusJob) Name() string { return jobNamePrometheus }
|
|
|
|
|
2018-09-23 21:08:03 +02:00
|
|
|
func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
|
2018-08-27 19:10:55 +02:00
|
|
|
|
2019-03-20 23:01:24 +01:00
|
|
|
func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
|
|
|
|
|
2018-09-08 07:03:41 +02:00
|
|
|
func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
|
|
|
|
|
2018-08-27 19:10:55 +02:00
|
|
|
func (j *prometheusJob) Run(ctx context.Context) {
|
|
|
|
|
|
|
|
if err := zfs.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
|
2018-12-11 22:01:50 +01:00
|
|
|
if err := frameconn.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
|
2018-08-27 19:10:55 +02:00
|
|
|
log := job.GetLogger(ctx)
|
|
|
|
|
|
|
|
l, err := net.Listen("tcp", j.listen)
|
|
|
|
if err != nil {
|
|
|
|
log.WithError(err).Error("cannot listen")
|
daemon/prometheus: fix crash if listener cannot be created
refs #238
zrepl version=v0.2.0-11-gdc39c81 GOOS=linux GOARCH=amd64 Compiler=gc
starting daemon
[pull_source]: starting job
[_prometheus]: starting job
[connection_loss_tidyup]: starting job
[connection_loss_tidyup]: wait for wakeups
[_control]: starting job
[_prometheus]: cannot listen err="listen tcp 10.0.0.200:9091: bind: cannot assign requested add
[_prometheus]: job exited
panic: runtime error: invalid memory address or nil pointer dereference
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x81ea4d]
goroutine 25 [running]:
net/http.(*onceCloseListener).close(...)
/usr/local/go/src/net/http/server.go:3330
sync.(*Once).doSlow(0xc00018b060, 0xc0000c7bc0)
/usr/local/go/src/sync/once.go:66 +0xe3
sync.(*Once).Do(...)
/usr/local/go/src/sync/once.go:57
net/http.(*onceCloseListener).Close(0xc00018b050, 0xc0003a6000, 0xe0)
/usr/local/go/src/net/http/server.go:3326 +0x77
panic(0xb1d1c0, 0x11d7d90)
/usr/local/go/src/runtime/panic.go:679 +0x1b2
net/http.(*onceCloseListener).Accept(0xc00018b050, 0xc000120020, 0xb0fd20, 0x11d7ce0, 0xbee6e0)
<autogenerated>:1 +0x32
net/http.(*Server).Serve(0xc0003a6000, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2896 +0x286
net/http.Serve(...)
/usr/local/go/src/net/http/server.go:2468
github.com/zrepl/zrepl/daemon.(*prometheusJob).Run(0xc000109940, 0xd19ec0, 0xc00018a930)
/go/src/github.com/zrepl/zrepl/daemon/prometheus.go:75 +0x23e
github.com/zrepl/zrepl/daemon.(*jobs).start.func1(0xc0000f68c0, 0xd22c40, 0xc000116ee0, 0xd1ba4
/go/src/github.com/zrepl/zrepl/daemon/daemon.go:220 +0x121
created by github.com/zrepl/zrepl/daemon.(*jobs).start
/go/src/github.com/zrepl/zrepl/daemon/daemon.go:216 +0x52e
zrepl.service: Main process exited, code=exited, status=2/INVALIDARGUMENT
zrepl.service: Failed with result 'exit-code'.
2019-10-30 10:57:40 +01:00
|
|
|
return
|
2018-08-27 19:10:55 +02:00
|
|
|
}
|
|
|
|
go func() {
|
2019-03-22 20:45:27 +01:00
|
|
|
<-ctx.Done()
|
|
|
|
l.Close()
|
2018-08-27 19:10:55 +02:00
|
|
|
}()
|
|
|
|
|
|
|
|
mux := http.NewServeMux()
|
|
|
|
mux.Handle("/metrics", promhttp.Handler())
|
|
|
|
|
|
|
|
err = http.Serve(l, mux)
|
2019-03-22 20:45:27 +01:00
|
|
|
if err != nil && ctx.Err() == nil {
|
2018-08-27 19:10:55 +02:00
|
|
|
log.WithError(err).Error("error while serving")
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
2018-09-08 07:03:41 +02:00
|
|
|
|
|
|
|
type prometheusJobOutlet struct {
|
|
|
|
jobName string
|
|
|
|
}
|
|
|
|
|
|
|
|
var _ logger.Outlet = prometheusJobOutlet{}
|
|
|
|
|
|
|
|
func newPrometheusLogOutlet(jobName string) prometheusJobOutlet {
|
|
|
|
return prometheusJobOutlet{jobName}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
|
|
|
|
prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
|
|
|
|
return nil
|
|
|
|
}
|