diff --git a/management/cmd/management.go b/management/cmd/management.go index fe187fb4a..e46493d97 100644 --- a/management/cmd/management.go +++ b/management/cmd/management.go @@ -8,14 +8,8 @@ import ( "flag" "fmt" "github.com/google/uuid" - "github.com/gorilla/mux" httpapi "github.com/netbirdio/netbird/management/server/http" "github.com/netbirdio/netbird/management/server/metrics" - prometheus2 "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "go.opentelemetry.io/otel/exporters/prometheus" - metric2 "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/sdk/metric" "golang.org/x/crypto/acme/autocert" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" @@ -26,7 +20,6 @@ import ( "net/url" "os" "path" - "reflect" "strings" "time" @@ -162,18 +155,17 @@ var ( gRPCOpts = append(gRPCOpts, grpc.Creds(transportCredentials)) tlsEnabled = true } - - metricsListener, err := net.Listen("tcp4", fmt.Sprintf(":%d", mgmtMetricsPort)) + appMetrics, err := metrics.NewDefaultAppMetrics(cmd.Context()) if err != nil { return err } - meter, err := exposeMetrics(metricsListener) + err = appMetrics.Expose(mgmtMetricsPort, "/metrics") if err != nil { return err } - httpAPIHandler, err := httpapi.APIHandler(accountManager, - config.HttpConfig.AuthIssuer, config.HttpConfig.AuthAudience, config.HttpConfig.AuthKeysLocation, meter) + httpAPIHandler, err := httpapi.APIHandler(cmd.Context(), accountManager, config.HttpConfig.AuthIssuer, + config.HttpConfig.AuthAudience, config.HttpConfig.AuthKeysLocation, appMetrics) if err != nil { return fmt.Errorf("failed creating HTTP API handler: %v", err) } @@ -246,7 +238,7 @@ var ( SetupCloseHandler() <-stopCh - _ = metricsListener.Close() + _ = appMetrics.Close() _ = listener.Close() if certManager != nil { _ = certManager.Listener().Close() @@ -259,29 +251,6 @@ var ( } ) -func exposeMetrics(lis net.Listener) (metric2.Meter, error) { - exporter, err := prometheus.New() - if err != nil { - return nil, err - } - pkg := reflect.TypeOf(ManagementLegacyPort).PkgPath() - provider := metric.NewMeterProvider(metric.WithReader(exporter)) - meter := provider.Meter(pkg) - rootRouter := mux.NewRouter() - rootRouter.Handle("/metrics", promhttp.HandlerFor( - prometheus2.DefaultGatherer, - promhttp.HandlerOpts{EnableOpenMetrics: true})) - - go func() { - err := http.Serve(lis, rootRouter) - if err != nil { - return - } - }() - log.Infof("metrics enabled for package %v and listening on %s", pkg, lis.Addr().String()) - return meter, nil -} - func notifyStop(msg string) { select { case stopCh <- 1: @@ -472,7 +441,7 @@ func loadTLSConfig(certFile string, certKey string) (*tls.Config, error) { return nil, err } - // Create the credentials and return it + // NewDefaultAppMetrics the credentials and return it config := &tls.Config{ Certificates: []tls.Certificate{serverCert}, ClientAuth: tls.NoClientCert, diff --git a/management/server/http/handler.go b/management/server/http/handler.go index a5ddd5163..16a21a154 100644 --- a/management/server/http/handler.go +++ b/management/server/http/handler.go @@ -5,14 +5,14 @@ import ( "github.com/gorilla/mux" s "github.com/netbirdio/netbird/management/server" "github.com/netbirdio/netbird/management/server/http/middleware" + "github.com/netbirdio/netbird/management/server/metrics" "github.com/rs/cors" - "go.opentelemetry.io/otel/metric" "net/http" ) // APIHandler creates the Management service HTTP API handler registering all the available endpoints. -func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience string, authKeysLocation string, - meter metric.Meter) (http.Handler, error) { +func APIHandler(ctx context.Context, accountManager s.AccountManager, authIssuer string, authAudience string, authKeysLocation string, + appMetrics metrics.AppMetrics) (http.Handler, error) { jwtMiddleware, err := middleware.NewJwtMiddleware( authIssuer, authAudience, @@ -29,13 +29,13 @@ func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience accountManager.IsUserAdmin) rootRouter := mux.NewRouter() - metrics, err := middleware.NewMetricsMiddleware(context.Background(), meter) + metricsMiddleware, err := metrics.NewMetricsMiddleware(ctx, appMetrics) if err != nil { return nil, err } apiHandler := rootRouter.PathPrefix("/api").Subrouter() - apiHandler.Use(metrics.Handler, corsMiddleware.Handler, jwtMiddleware.Handler, acMiddleware.Handler) + apiHandler.Use(metricsMiddleware.Handler, corsMiddleware.Handler, jwtMiddleware.Handler, acMiddleware.Handler) groupsHandler := NewGroups(accountManager, authAudience) rulesHandler := NewRules(accountManager, authAudience) @@ -95,7 +95,7 @@ func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience if err != nil { return err } - err = metrics.AddHTTPRequestResponseCounter(template, method) + err = metricsMiddleware.AddHTTPRequestResponseCounter(template, method) if err != nil { return err } diff --git a/management/server/metrics/app.go b/management/server/metrics/app.go new file mode 100644 index 000000000..454c86e04 --- /dev/null +++ b/management/server/metrics/app.go @@ -0,0 +1,87 @@ +package metrics + +import ( + "context" + "fmt" + "github.com/gorilla/mux" + prometheus2 "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/exporters/prometheus" + metric2 "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/sdk/metric" + "net" + "net/http" + "reflect" +) + +const defaultEndpoint = "/metrics" + +// AppMetrics is metrics interface +type AppMetrics interface { + GetMeter() metric2.Meter + Close() error + Expose(port int, endpoint string) error +} + +// defaultAppMetrics are core application metrics based on OpenTelemetry https://opentelemetry.io/ +type defaultAppMetrics struct { + // Meter can be used by different application parts to create counters and measure things + Meter metric2.Meter + listener net.Listener + ctx context.Context +} + +// Close stop application metrics HTTP handler and closes listener. +func (appMetrics *defaultAppMetrics) Close() error { + if appMetrics.listener == nil { + return nil + } + return appMetrics.listener.Close() +} + +// Expose metrics on a given port and endpoint. If endpoint is empty a defaultEndpoint one will be used. +// Exposes metrics in the Prometheus format https://prometheus.io/ +func (appMetrics *defaultAppMetrics) Expose(port int, endpoint string) error { + if endpoint == "" { + endpoint = defaultEndpoint + } + rootRouter := mux.NewRouter() + rootRouter.Handle(endpoint, promhttp.HandlerFor( + prometheus2.DefaultGatherer, + promhttp.HandlerOpts{EnableOpenMetrics: true})) + listener, err := net.Listen("tcp4", fmt.Sprintf(":%d", port)) + if err != nil { + return err + } + appMetrics.listener = listener + go func() { + err := http.Serve(listener, rootRouter) + if err != nil { + return + } + }() + + log.Infof("enabled application metrics and exposing on http://%s", listener.Addr().String()) + + return nil +} + +// GetMeter returns metrics meter that can be used to add various counters +func (appMetrics *defaultAppMetrics) GetMeter() metric2.Meter { + return appMetrics.Meter +} + +// NewDefaultAppMetrics and expose them via defaultEndpoint on a given HTTP port +func NewDefaultAppMetrics(ctx context.Context) (AppMetrics, error) { + exporter, err := prometheus.New() + if err != nil { + return nil, err + } + + provider := metric.NewMeterProvider(metric.WithReader(exporter)) + pkg := reflect.TypeOf(defaultEndpoint).PkgPath() + meter := provider.Meter(pkg) + + return &defaultAppMetrics{Meter: meter, ctx: ctx}, nil +} diff --git a/management/server/http/middleware/metrics.go b/management/server/metrics/middleware.go similarity index 77% rename from management/server/http/middleware/metrics.go rename to management/server/metrics/middleware.go index c2cfef613..a347c7bd9 100644 --- a/management/server/http/middleware/metrics.go +++ b/management/server/metrics/middleware.go @@ -1,10 +1,9 @@ -package middleware +package metrics import ( "context" "fmt" log "github.com/sirupsen/logrus" - metric2 "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/metric/instrument" "go.opentelemetry.io/otel/metric/instrument/syncint64" "hash/fnv" @@ -46,14 +45,14 @@ func (rw *WrappedResponseWriter) WriteHeader(code int) { rw.wroteHeader = true } -// MetricsMiddleware handler used to collect metrics of every request/response coming to the API. +// HTTPMiddleware handler used to collect metrics of every request/response coming to the API. // Also adds request tracing (logging). -type MetricsMiddleware struct { - meter metric2.Meter - ctx context.Context - // endpoint & method +type HTTPMiddleware struct { + appMetrics AppMetrics + ctx context.Context + // defaultEndpoint & method httpRequestCounters map[string]syncint64.Counter - // endpoint & method & status code + // defaultEndpoint & method & status code httpResponseCounters map[string]syncint64.Counter // all HTTP requests totalHTTPRequestsCounter syncint64.Counter @@ -63,11 +62,11 @@ type MetricsMiddleware struct { totalHTTPResponseCodeCounters map[int]syncint64.Counter } -// AddHTTPRequestResponseCounter adds a new meter for an HTTP endpoint and Method (GET, POST, etc) +// AddHTTPRequestResponseCounter adds a new meter for an HTTP defaultEndpoint and Method (GET, POST, etc) // Creates one request counter and multiple response counters (one per http response status code). -func (m *MetricsMiddleware) AddHTTPRequestResponseCounter(endpoint string, method string) error { +func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method string) error { meterKey := getRequestCounterKey(endpoint, method) - httpReqCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1")) + httpReqCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1")) if err != nil { return err } @@ -75,14 +74,14 @@ func (m *MetricsMiddleware) AddHTTPRequestResponseCounter(endpoint string, metho respCodes := []int{200, 204, 400, 401, 403, 404, 500, 502, 503} for _, code := range respCodes { meterKey = getResponseCounterKey(endpoint, method, code) - httpRespCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1")) + httpRespCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1")) if err != nil { return err } m.httpResponseCounters[meterKey] = httpRespCounter meterKey = fmt.Sprintf("%s_%d_total", httpResponseCounterPrefix, code) - totalHTTPResponseCodeCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1")) + totalHTTPResponseCodeCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1")) if err != nil { return err } @@ -92,27 +91,27 @@ func (m *MetricsMiddleware) AddHTTPRequestResponseCounter(endpoint string, metho return nil } -// NewMetricsMiddleware creates a new MetricsMiddleware -func NewMetricsMiddleware(ctx context.Context, meter metric2.Meter) (*MetricsMiddleware, error) { +// NewMetricsMiddleware creates a new HTTPMiddleware +func NewMetricsMiddleware(ctx context.Context, appMetrics AppMetrics) (*HTTPMiddleware, error) { - totalHTTPRequestsCounter, err := meter.SyncInt64().Counter( + totalHTTPRequestsCounter, err := appMetrics.GetMeter().SyncInt64().Counter( fmt.Sprintf("%s_total", httpRequestCounterPrefix), instrument.WithUnit("1")) if err != nil { return nil, err } - totalHTTPResponseCounter, err := meter.SyncInt64().Counter( + totalHTTPResponseCounter, err := appMetrics.GetMeter().SyncInt64().Counter( fmt.Sprintf("%s_total", httpResponseCounterPrefix), instrument.WithUnit("1")) if err != nil { return nil, err } - return &MetricsMiddleware{ + return &HTTPMiddleware{ ctx: ctx, httpRequestCounters: map[string]syncint64.Counter{}, httpResponseCounters: map[string]syncint64.Counter{}, totalHTTPResponseCodeCounters: map[int]syncint64.Counter{}, - meter: meter, + appMetrics: appMetrics, totalHTTPRequestsCounter: totalHTTPRequestsCounter, totalHTTPResponseCounter: totalHTTPResponseCounter, }, @@ -130,7 +129,7 @@ func getResponseCounterKey(endpoint, method string, status int) string { } // Handler logs every request and response and adds the, to metrics. -func (m *MetricsMiddleware) Handler(h http.Handler) http.Handler { +func (m *HTTPMiddleware) Handler(h http.Handler) http.Handler { fn := func(rw http.ResponseWriter, r *http.Request) { traceID := hash(fmt.Sprintf("%v", r)) log.Tracef("HTTP request %v: %v %v", traceID, r.Method, r.URL) diff --git a/management/server/metrics/metrics.go b/management/server/metrics/selfhosted.go similarity index 99% rename from management/server/metrics/metrics.go rename to management/server/metrics/selfhosted.go index d7287efd4..40be956c4 100644 --- a/management/server/metrics/metrics.go +++ b/management/server/metrics/selfhosted.go @@ -17,7 +17,7 @@ import ( const ( // PayloadEvent identifies an event type PayloadEvent = "self-hosted stats" - // payloadEndpoint metrics endpoint to send anonymous data + // payloadEndpoint metrics defaultEndpoint to send anonymous data payloadEndpoint = "https://metrics.netbird.io" // defaultPushInterval default interval to push metrics defaultPushInterval = 24 * time.Hour