Extract app metrics to a separate struct (#520)

management/cmd/management.go

@@ -8,14 +8,8 @@ import (
 	"flag"
 	"fmt"
 	"github.com/google/uuid"
-	"github.com/gorilla/mux"
 	httpapi "github.com/netbirdio/netbird/management/server/http"
 	"github.com/netbirdio/netbird/management/server/metrics"
-	prometheus2 "github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"go.opentelemetry.io/otel/exporters/prometheus"
-	metric2 "go.opentelemetry.io/otel/metric"
-	"go.opentelemetry.io/otel/sdk/metric"
 	"golang.org/x/crypto/acme/autocert"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
@@ -26,7 +20,6 @@ import (
 	"net/url"
 	"os"
 	"path"
-	"reflect"
 	"strings"
 	"time"
 
@@ -162,18 +155,17 @@ var (
 				gRPCOpts = append(gRPCOpts, grpc.Creds(transportCredentials))
 				tlsEnabled = true
 			}
-
-			metricsListener, err := net.Listen("tcp4", fmt.Sprintf(":%d", mgmtMetricsPort))
+			appMetrics, err := metrics.NewDefaultAppMetrics(cmd.Context())
 			if err != nil {
 				return err
 			}
-			meter, err := exposeMetrics(metricsListener)
+			err = appMetrics.Expose(mgmtMetricsPort, "/metrics")
 			if err != nil {
 				return err
 			}
 
-			httpAPIHandler, err := httpapi.APIHandler(accountManager,
-				config.HttpConfig.AuthIssuer, config.HttpConfig.AuthAudience, config.HttpConfig.AuthKeysLocation, meter)
+			httpAPIHandler, err := httpapi.APIHandler(cmd.Context(), accountManager, config.HttpConfig.AuthIssuer,
+				config.HttpConfig.AuthAudience, config.HttpConfig.AuthKeysLocation, appMetrics)
 			if err != nil {
 				return fmt.Errorf("failed creating HTTP API handler: %v", err)
 			}
@@ -246,7 +238,7 @@ var (
 			SetupCloseHandler()
 
 			<-stopCh
-			_ = metricsListener.Close()
+			_ = appMetrics.Close()
 			_ = listener.Close()
 			if certManager != nil {
 				_ = certManager.Listener().Close()
@@ -259,29 +251,6 @@ var (
 	}
 )
 
-func exposeMetrics(lis net.Listener) (metric2.Meter, error) {
-	exporter, err := prometheus.New()
-	if err != nil {
-		return nil, err
-	}
-	pkg := reflect.TypeOf(ManagementLegacyPort).PkgPath()
-	provider := metric.NewMeterProvider(metric.WithReader(exporter))
-	meter := provider.Meter(pkg)
-	rootRouter := mux.NewRouter()
-	rootRouter.Handle("/metrics", promhttp.HandlerFor(
-		prometheus2.DefaultGatherer,
-		promhttp.HandlerOpts{EnableOpenMetrics: true}))
-
-	go func() {
-		err := http.Serve(lis, rootRouter)
-		if err != nil {
-			return
-		}
-	}()
-	log.Infof("metrics enabled for package %v and listening on %s", pkg, lis.Addr().String())
-	return meter, nil
-}
-
 func notifyStop(msg string) {
 	select {
 	case stopCh <- 1:
@@ -472,7 +441,7 @@ func loadTLSConfig(certFile string, certKey string) (*tls.Config, error) {
 		return nil, err
 	}
 
-	// Create the credentials and return it
+	// NewDefaultAppMetrics the credentials and return it
 	config := &tls.Config{
 		Certificates: []tls.Certificate{serverCert},
 		ClientAuth:   tls.NoClientCert,

management/server/http/handler.go

@@ -5,14 +5,14 @@ import (
 	"github.com/gorilla/mux"
 	s "github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/http/middleware"
+	"github.com/netbirdio/netbird/management/server/metrics"
 	"github.com/rs/cors"
-	"go.opentelemetry.io/otel/metric"
 	"net/http"
 )
 
 // APIHandler creates the Management service HTTP API handler registering all the available endpoints.
-func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience string, authKeysLocation string,
-	meter metric.Meter) (http.Handler, error) {
+func APIHandler(ctx context.Context, accountManager s.AccountManager, authIssuer string, authAudience string, authKeysLocation string,
+	appMetrics metrics.AppMetrics) (http.Handler, error) {
 	jwtMiddleware, err := middleware.NewJwtMiddleware(
 		authIssuer,
 		authAudience,
@@ -29,13 +29,13 @@ func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience
 		accountManager.IsUserAdmin)
 
 	rootRouter := mux.NewRouter()
-	metrics, err := middleware.NewMetricsMiddleware(context.Background(), meter)
+	metricsMiddleware, err := metrics.NewMetricsMiddleware(ctx, appMetrics)
 	if err != nil {
 		return nil, err
 	}
 
 	apiHandler := rootRouter.PathPrefix("/api").Subrouter()
-	apiHandler.Use(metrics.Handler, corsMiddleware.Handler, jwtMiddleware.Handler, acMiddleware.Handler)
+	apiHandler.Use(metricsMiddleware.Handler, corsMiddleware.Handler, jwtMiddleware.Handler, acMiddleware.Handler)
 
 	groupsHandler := NewGroups(accountManager, authAudience)
 	rulesHandler := NewRules(accountManager, authAudience)
@@ -95,7 +95,7 @@ func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience
 			if err != nil {
 				return err
 			}
-			err = metrics.AddHTTPRequestResponseCounter(template, method)
+			err = metricsMiddleware.AddHTTPRequestResponseCounter(template, method)
 			if err != nil {
 				return err
 			}

management/server/metrics/app.go

@@ -0,0 +1,87 @@
+package metrics
+
+import (
+	"context"
+	"fmt"
+	"github.com/gorilla/mux"
+	prometheus2 "github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	metric2 "go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"net"
+	"net/http"
+	"reflect"
+)
+
+const defaultEndpoint = "/metrics"
+
+// AppMetrics is metrics interface
+type AppMetrics interface {
+	GetMeter() metric2.Meter
+	Close() error
+	Expose(port int, endpoint string) error
+}
+
+// defaultAppMetrics are core application metrics based on OpenTelemetry https://opentelemetry.io/
+type defaultAppMetrics struct {
+	// Meter can be used by different application parts to create counters and measure things
+	Meter    metric2.Meter
+	listener net.Listener
+	ctx      context.Context
+}
+
+// Close stop application metrics HTTP handler and closes listener.
+func (appMetrics *defaultAppMetrics) Close() error {
+	if appMetrics.listener == nil {
+		return nil
+	}
+	return appMetrics.listener.Close()
+}
+
+// Expose metrics on a given port and endpoint. If endpoint is empty a defaultEndpoint one will be used.
+// Exposes metrics in the Prometheus format https://prometheus.io/
+func (appMetrics *defaultAppMetrics) Expose(port int, endpoint string) error {
+	if endpoint == "" {
+		endpoint = defaultEndpoint
+	}
+	rootRouter := mux.NewRouter()
+	rootRouter.Handle(endpoint, promhttp.HandlerFor(
+		prometheus2.DefaultGatherer,
+		promhttp.HandlerOpts{EnableOpenMetrics: true}))
+	listener, err := net.Listen("tcp4", fmt.Sprintf(":%d", port))
+	if err != nil {
+		return err
+	}
+	appMetrics.listener = listener
+	go func() {
+		err := http.Serve(listener, rootRouter)
+		if err != nil {
+			return
+		}
+	}()
+
+	log.Infof("enabled application metrics and exposing on http://%s", listener.Addr().String())
+
+	return nil
+}
+
+// GetMeter returns metrics meter that can be used to add various counters
+func (appMetrics *defaultAppMetrics) GetMeter() metric2.Meter {
+	return appMetrics.Meter
+}
+
+// NewDefaultAppMetrics and expose them via defaultEndpoint on a given HTTP port
+func NewDefaultAppMetrics(ctx context.Context) (AppMetrics, error) {
+	exporter, err := prometheus.New()
+	if err != nil {
+		return nil, err
+	}
+
+	provider := metric.NewMeterProvider(metric.WithReader(exporter))
+	pkg := reflect.TypeOf(defaultEndpoint).PkgPath()
+	meter := provider.Meter(pkg)
+
+	return &defaultAppMetrics{Meter: meter, ctx: ctx}, nil
+}

management/server/metrics/middleware.go

@@ -1,10 +1,9 @@
-package middleware
+package metrics
 
 import (
 	"context"
 	"fmt"
 	log "github.com/sirupsen/logrus"
-	metric2 "go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/metric/instrument"
 	"go.opentelemetry.io/otel/metric/instrument/syncint64"
 	"hash/fnv"
@@ -46,14 +45,14 @@ func (rw *WrappedResponseWriter) WriteHeader(code int) {
 	rw.wroteHeader = true
 }
 
-// MetricsMiddleware handler used to collect metrics of every request/response coming to the API.
+// HTTPMiddleware handler used to collect metrics of every request/response coming to the API.
 // Also adds request tracing (logging).
-type MetricsMiddleware struct {
-	meter metric2.Meter
-	ctx   context.Context
-	// endpoint & method
+type HTTPMiddleware struct {
+	appMetrics AppMetrics
+	ctx        context.Context
+	// defaultEndpoint & method
 	httpRequestCounters map[string]syncint64.Counter
-	// endpoint & method & status code
+	// defaultEndpoint & method & status code
 	httpResponseCounters map[string]syncint64.Counter
 	// all HTTP requests
 	totalHTTPRequestsCounter syncint64.Counter
@@ -63,11 +62,11 @@ type MetricsMiddleware struct {
 	totalHTTPResponseCodeCounters map[int]syncint64.Counter
 }
 
-// AddHTTPRequestResponseCounter adds a new meter for an HTTP endpoint and Method (GET, POST, etc)
+// AddHTTPRequestResponseCounter adds a new meter for an HTTP defaultEndpoint and Method (GET, POST, etc)
 // Creates one request counter and multiple response counters (one per http response status code).
-func (m *MetricsMiddleware) AddHTTPRequestResponseCounter(endpoint string, method string) error {
+func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method string) error {
 	meterKey := getRequestCounterKey(endpoint, method)
-	httpReqCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+	httpReqCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
 	if err != nil {
 		return err
 	}
@@ -75,14 +74,14 @@ func (m *MetricsMiddleware) AddHTTPRequestResponseCounter(endpoint string, metho
 	respCodes := []int{200, 204, 400, 401, 403, 404, 500, 502, 503}
 	for _, code := range respCodes {
 		meterKey = getResponseCounterKey(endpoint, method, code)
-		httpRespCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+		httpRespCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
 		if err != nil {
 			return err
 		}
 		m.httpResponseCounters[meterKey] = httpRespCounter
 
 		meterKey = fmt.Sprintf("%s_%d_total", httpResponseCounterPrefix, code)
-		totalHTTPResponseCodeCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+		totalHTTPResponseCodeCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
 		if err != nil {
 			return err
 		}
@@ -92,27 +91,27 @@ func (m *MetricsMiddleware) AddHTTPRequestResponseCounter(endpoint string, metho
 	return nil
 }
 
-// NewMetricsMiddleware creates a new MetricsMiddleware
-func NewMetricsMiddleware(ctx context.Context, meter metric2.Meter) (*MetricsMiddleware, error) {
+// NewMetricsMiddleware creates a new HTTPMiddleware
+func NewMetricsMiddleware(ctx context.Context, appMetrics AppMetrics) (*HTTPMiddleware, error) {
 
-	totalHTTPRequestsCounter, err := meter.SyncInt64().Counter(
+	totalHTTPRequestsCounter, err := appMetrics.GetMeter().SyncInt64().Counter(
 		fmt.Sprintf("%s_total", httpRequestCounterPrefix),
 		instrument.WithUnit("1"))
 	if err != nil {
 		return nil, err
 	}
-	totalHTTPResponseCounter, err := meter.SyncInt64().Counter(
+	totalHTTPResponseCounter, err := appMetrics.GetMeter().SyncInt64().Counter(
 		fmt.Sprintf("%s_total", httpResponseCounterPrefix),
 		instrument.WithUnit("1"))
 	if err != nil {
 		return nil, err
 	}
-	return &MetricsMiddleware{
+	return &HTTPMiddleware{
 			ctx:                           ctx,
 			httpRequestCounters:           map[string]syncint64.Counter{},
 			httpResponseCounters:          map[string]syncint64.Counter{},
 			totalHTTPResponseCodeCounters: map[int]syncint64.Counter{},
-			meter:                         meter,
+			appMetrics:                    appMetrics,
 			totalHTTPRequestsCounter:      totalHTTPRequestsCounter,
 			totalHTTPResponseCounter:      totalHTTPResponseCounter,
 		},
@@ -130,7 +129,7 @@ func getResponseCounterKey(endpoint, method string, status int) string {
 }
 
 // Handler logs every request and response and adds the, to metrics.
-func (m *MetricsMiddleware) Handler(h http.Handler) http.Handler {
+func (m *HTTPMiddleware) Handler(h http.Handler) http.Handler {
 	fn := func(rw http.ResponseWriter, r *http.Request) {
 		traceID := hash(fmt.Sprintf("%v", r))
 		log.Tracef("HTTP request %v: %v %v", traceID, r.Method, r.URL)

management/server/metrics/selfhosted.go

@@ -17,7 +17,7 @@ import (
 const (
 	// PayloadEvent identifies an event type
 	PayloadEvent = "self-hosted stats"
-	// payloadEndpoint metrics endpoint to send anonymous data
+	// payloadEndpoint metrics defaultEndpoint to send anonymous data
 	payloadEndpoint = "https://metrics.netbird.io"
 	// defaultPushInterval default interval to push metrics
 	defaultPushInterval = 24 * time.Hour