mirror of
https://github.com/zrepl/zrepl.git
synced 2025-06-21 18:21:24 +02:00
daemon control / jsonclient: fix connection leak due to open request body
Also: - Defensive measures in control http server (1s timeouts) (prevent the leak, even if request body is not closed) - Add prometheus metrics to track control socket latencies (were used for debugging)
This commit is contained in:
parent
fa47667f31
commit
6c3f442f13
@ -30,7 +30,10 @@ func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res in
|
|||||||
resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
|
resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
} else if resp.StatusCode != http.StatusOK {
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
var msg bytes.Buffer
|
var msg bytes.Buffer
|
||||||
io.CopyN(&msg, resp.Body, 4096)
|
io.CopyN(&msg, resp.Body, 4096)
|
||||||
return errors.Errorf("%s", msg.String())
|
return errors.Errorf("%s", msg.String())
|
||||||
|
@ -13,6 +13,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
type controlJob struct {
|
type controlJob struct {
|
||||||
@ -36,7 +37,29 @@ func (j *controlJob) Name() string { return jobNameControl }
|
|||||||
|
|
||||||
func (j *controlJob) Status() interface{} { return nil }
|
func (j *controlJob) Status() interface{} { return nil }
|
||||||
|
|
||||||
func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {}
|
var promControl struct {
|
||||||
|
requestBegin *prometheus.CounterVec
|
||||||
|
requestFinished *prometheus.HistogramVec
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
|
||||||
|
promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||||
|
Namespace: "zrepl",
|
||||||
|
Subsystem: "control",
|
||||||
|
Name: "request_begin",
|
||||||
|
Help: "number of request we started to handle",
|
||||||
|
}, []string{"endpoint"})
|
||||||
|
|
||||||
|
promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||||
|
Namespace: "zrepl",
|
||||||
|
Subsystem: "control",
|
||||||
|
Name: "request_finished",
|
||||||
|
Help: "time it took a request to finih",
|
||||||
|
Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3,10e-3, 100e-3, 200e-3,400e-3,800e-3, 1, 10, 20},
|
||||||
|
}, []string{"endpoint"})
|
||||||
|
registerer.MustRegister(promControl.requestBegin)
|
||||||
|
registerer.MustRegister(promControl.requestFinished)
|
||||||
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
ControlJobEndpointPProf string = "/debug/pprof"
|
ControlJobEndpointPProf string = "/debug/pprof"
|
||||||
@ -95,7 +118,12 @@ func (j *controlJob) Run(ctx context.Context) {
|
|||||||
|
|
||||||
return struct{}{}, err
|
return struct{}{}, err
|
||||||
}}})
|
}}})
|
||||||
server := http.Server{Handler: mux}
|
server := http.Server{
|
||||||
|
Handler: mux,
|
||||||
|
// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
|
||||||
|
WriteTimeout: 1*time.Second,
|
||||||
|
ReadTimeout: 1*time.Second,
|
||||||
|
}
|
||||||
|
|
||||||
outer:
|
outer:
|
||||||
for {
|
for {
|
||||||
@ -189,6 +217,8 @@ type requestLogger struct {
|
|||||||
func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
log := l.log.WithField("method", r.Method).WithField("url", r.URL)
|
log := l.log.WithField("method", r.Method).WithField("url", r.URL)
|
||||||
log.Debug("start")
|
log.Debug("start")
|
||||||
|
promControl.requestBegin.WithLabelValues(r.URL.Path).Inc()
|
||||||
|
defer prometheus.NewTimer(promControl.requestFinished.WithLabelValues(r.URL.Path)).ObserveDuration()
|
||||||
if l.handlerFunc != nil {
|
if l.handlerFunc != nil {
|
||||||
l.handlerFunc(w, r)
|
l.handlerFunc(w, r)
|
||||||
} else if l.handler != nil {
|
} else if l.handler != nil {
|
||||||
|
Loading…
x
Reference in New Issue
Block a user