zrepl/daemon/prometheus.go
Christian Schwarz ad0b055245 daemon/prometheus: fix crash if listener cannot be created
refs #238

  zrepl version=v0.2.0-11-gdc39c81 GOOS=linux GOARCH=amd64 Compiler=gc
 starting daemon
 [pull_source]: starting job
 [_prometheus]: starting job
 [connection_loss_tidyup]: starting job
 [connection_loss_tidyup]: wait for wakeups
 [_control]: starting job
 [_prometheus]: cannot listen err="listen tcp 10.0.0.200:9091: bind: cannot assign requested add
 [_prometheus]: job exited
 panic: runtime error: invalid memory address or nil pointer dereference
         panic: runtime error: invalid memory address or nil pointer dereference
 [signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x81ea4d]
 goroutine 25 [running]:
 net/http.(*onceCloseListener).close(...)
         /usr/local/go/src/net/http/server.go:3330
 sync.(*Once).doSlow(0xc00018b060, 0xc0000c7bc0)
         /usr/local/go/src/sync/once.go:66 +0xe3
 sync.(*Once).Do(...)
         /usr/local/go/src/sync/once.go:57
 net/http.(*onceCloseListener).Close(0xc00018b050, 0xc0003a6000, 0xe0)
         /usr/local/go/src/net/http/server.go:3326 +0x77
 panic(0xb1d1c0, 0x11d7d90)
         /usr/local/go/src/runtime/panic.go:679 +0x1b2
 net/http.(*onceCloseListener).Accept(0xc00018b050, 0xc000120020, 0xb0fd20, 0x11d7ce0, 0xbee6e0)
         <autogenerated>:1 +0x32
 net/http.(*Server).Serve(0xc0003a6000, 0x0, 0x0, 0x0, 0x0)
         /usr/local/go/src/net/http/server.go:2896 +0x286
 net/http.Serve(...)
         /usr/local/go/src/net/http/server.go:2468
 github.com/zrepl/zrepl/daemon.(*prometheusJob).Run(0xc000109940, 0xd19ec0, 0xc00018a930)
         /go/src/github.com/zrepl/zrepl/daemon/prometheus.go:75 +0x23e
 github.com/zrepl/zrepl/daemon.(*jobs).start.func1(0xc0000f68c0, 0xd22c40, 0xc000116ee0, 0xd1ba4
         /go/src/github.com/zrepl/zrepl/daemon/daemon.go:220 +0x121
 created by github.com/zrepl/zrepl/daemon.(*jobs).start
         /go/src/github.com/zrepl/zrepl/daemon/daemon.go:216 +0x52e
 zrepl.service: Main process exited, code=exited, status=2/INVALIDARGUMENT
 zrepl.service: Failed with result 'exit-code'.
2019-11-16 22:11:13 +01:00

97 lines
2.3 KiB
Go

package daemon
import (
"context"
"net"
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/job"
"github.com/zrepl/zrepl/logger"
"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
"github.com/zrepl/zrepl/zfs"
)
type prometheusJob struct {
listen string
}
func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
if _, _, err := net.SplitHostPort(in.Listen); err != nil {
return nil, err
}
return &prometheusJob{in.Listen}, nil
}
var prom struct {
taskLogEntries *prometheus.CounterVec
}
func init() {
prom.taskLogEntries = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "zrepl",
Subsystem: "daemon",
Name: "log_entries",
Help: "number of log entries per job task and level",
}, []string{"zrepl_job", "level"})
prometheus.MustRegister(prom.taskLogEntries)
}
func (j *prometheusJob) Name() string { return jobNamePrometheus }
func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
func (j *prometheusJob) Run(ctx context.Context) {
if err := zfs.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
panic(err)
}
if err := frameconn.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
panic(err)
}
log := job.GetLogger(ctx)
l, err := net.Listen("tcp", j.listen)
if err != nil {
log.WithError(err).Error("cannot listen")
return
}
go func() {
<-ctx.Done()
l.Close()
}()
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
err = http.Serve(l, mux)
if err != nil && ctx.Err() == nil {
log.WithError(err).Error("error while serving")
}
}
type prometheusJobOutlet struct {
jobName string
}
var _ logger.Outlet = prometheusJobOutlet{}
func newPrometheusLogOutlet(jobName string) prometheusJobOutlet {
return prometheusJobOutlet{jobName}
}
func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
return nil
}