Add HTTP request/response totals to metrics (#519)

management/server/http/middleware/metrics.go

@@ -49,10 +49,18 @@ func (rw *WrappedResponseWriter) WriteHeader(code int) {
 // MetricsMiddleware handler used to collect metrics of every request/response coming to the API.
 // Also adds request tracing (logging).
 type MetricsMiddleware struct {
-	meter                metric2.Meter
+	meter metric2.Meter
-	ctx                  context.Context
+	ctx   context.Context
-	httpRequestCounters  map[string]syncint64.Counter
+	// endpoint & method
+	httpRequestCounters map[string]syncint64.Counter
+	// endpoint & method & status code
 	httpResponseCounters map[string]syncint64.Counter
+	// all HTTP requests
+	totalHTTPRequestsCounter syncint64.Counter
+	// all HTTP responses
+	totalHTTPResponseCounter syncint64.Counter
+	// all HTTP responses by status code
+	totalHTTPResponseCodeCounters map[int]syncint64.Counter
 }
 
 // AddHTTPRequestResponseCounter adds a new meter for an HTTP endpoint and Method (GET, POST, etc)
@@ -64,7 +72,7 @@ func (m *MetricsMiddleware) AddHTTPRequestResponseCounter(endpoint string, metho
 		return err
 	}
 	m.httpRequestCounters[meterKey] = httpReqCounter
-	respCodes := []int{200, 204, 400, 401, 403, 500, 502, 503}
+	respCodes := []int{200, 204, 400, 401, 403, 404, 500, 502, 503}
 	for _, code := range respCodes {
 		meterKey = getResponseCounterKey(endpoint, method, code)
 		httpRespCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
@@ -72,6 +80,13 @@ func (m *MetricsMiddleware) AddHTTPRequestResponseCounter(endpoint string, metho
 			return err
 		}
 		m.httpResponseCounters[meterKey] = httpRespCounter
+
+		meterKey = fmt.Sprintf("%s_%d_total", httpResponseCounterPrefix, code)
+		totalHTTPResponseCodeCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+		if err != nil {
+			return err
+		}
+		m.totalHTTPResponseCodeCounters[code] = totalHTTPResponseCodeCounter
 	}
 
 	return nil
@@ -79,11 +94,27 @@ func (m *MetricsMiddleware) AddHTTPRequestResponseCounter(endpoint string, metho
 
 // NewMetricsMiddleware creates a new MetricsMiddleware
 func NewMetricsMiddleware(ctx context.Context, meter metric2.Meter) (*MetricsMiddleware, error) {
+
+	totalHTTPRequestsCounter, err := meter.SyncInt64().Counter(
+		fmt.Sprintf("%s_total", httpRequestCounterPrefix),
+		instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	totalHTTPResponseCounter, err := meter.SyncInt64().Counter(
+		fmt.Sprintf("%s_total", httpResponseCounterPrefix),
+		instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
 	return &MetricsMiddleware{
-			ctx:                  ctx,
+			ctx:                           ctx,
-			httpRequestCounters:  map[string]syncint64.Counter{},
+			httpRequestCounters:           map[string]syncint64.Counter{},
-			httpResponseCounters: map[string]syncint64.Counter{},
+			httpResponseCounters:          map[string]syncint64.Counter{},
-			meter:                meter,
+			totalHTTPResponseCodeCounters: map[int]syncint64.Counter{},
+			meter:                         meter,
+			totalHTTPRequestsCounter:      totalHTTPRequestsCounter,
+			totalHTTPResponseCounter:      totalHTTPResponseCounter,
 		},
 		nil
 }
@@ -109,6 +140,7 @@ func (m *MetricsMiddleware) Handler(h http.Handler) http.Handler {
 		if c, ok := m.httpRequestCounters[metricKey]; ok {
 			c.Add(m.ctx, 1)
 		}
+		m.totalHTTPRequestsCounter.Add(m.ctx, 1)
 
 		w := WrapResponseWriter(rw)
 
@@ -124,6 +156,11 @@ func (m *MetricsMiddleware) Handler(h http.Handler) http.Handler {
 		if c, ok := m.httpResponseCounters[metricKey]; ok {
 			c.Add(m.ctx, 1)
 		}
+
+		m.totalHTTPResponseCounter.Add(m.ctx, 1)
+		if c, ok := m.totalHTTPResponseCodeCounters[w.Status()]; ok {
+			c.Add(m.ctx, 1)
+		}
 	}
 
 	return http.HandlerFunc(fn)