From 830dee177128fa374c0138e9f3838e1f0a2bd7e0 Mon Sep 17 00:00:00 2001
From: Yury Gargay <yury.gargay@gmail.com>
Date: Thu, 28 Sep 2023 08:54:49 +0200
Subject: [PATCH] Expose store metrics with milliseconds bucketing (#1179)

As the current upper 10000 microseconds(10ms) bucket may be too low for
`management.store.persistence.duration` metric
---
 management/server/telemetry/store_metrics.go | 37 ++++++++++++++------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/management/server/telemetry/store_metrics.go b/management/server/telemetry/store_metrics.go
index 704ef65d4..98c13f12a 100644
--- a/management/server/telemetry/store_metrics.go
+++ b/management/server/telemetry/store_metrics.go
@@ -11,37 +11,54 @@ import (
 
 // StoreMetrics represents all metrics related to the FileStore
 type StoreMetrics struct {
-	globalLockAcquisitionDuration syncint64.Histogram
-	persistenceDuration           syncint64.Histogram
-	ctx                           context.Context
+	globalLockAcquisitionDurationMicro syncint64.Histogram
+	globalLockAcquisitionDurationMs    syncint64.Histogram
+	persistenceDurationMicro           syncint64.Histogram
+	persistenceDurationMs              syncint64.Histogram
+	ctx                                context.Context
 }
 
 // NewStoreMetrics creates an instance of StoreMetrics
 func NewStoreMetrics(ctx context.Context, meter metric.Meter) (*StoreMetrics, error) {
-	globalLockAcquisitionDuration, err := meter.SyncInt64().Histogram("management.store.global.lock.acquisition.duration.micro",
+	globalLockAcquisitionDurationMicro, err := meter.SyncInt64().Histogram("management.store.global.lock.acquisition.duration.micro",
 		instrument.WithUnit("microseconds"))
 	if err != nil {
 		return nil, err
 	}
-	persistenceDuration, err := meter.SyncInt64().Histogram("management.store.persistence.duration.micro",
+
+	globalLockAcquisitionDurationMs, err := meter.SyncInt64().Histogram("management.store.global.lock.acquisition.duration.ms")
+	if err != nil {
+		return nil, err
+	}
+
+	persistenceDurationMicro, err := meter.SyncInt64().Histogram("management.store.persistence.duration.micro",
 		instrument.WithUnit("microseconds"))
 	if err != nil {
 		return nil, err
 	}
 
+	persistenceDurationMs, err := meter.SyncInt64().Histogram("management.store.persistence.duration.ms")
+	if err != nil {
+		return nil, err
+	}
+
 	return &StoreMetrics{
-		globalLockAcquisitionDuration: globalLockAcquisitionDuration,
-		persistenceDuration:           persistenceDuration,
-		ctx:                           ctx,
+		globalLockAcquisitionDurationMicro: globalLockAcquisitionDurationMicro,
+		globalLockAcquisitionDurationMs:    globalLockAcquisitionDurationMs,
+		persistenceDurationMicro:           persistenceDurationMicro,
+		persistenceDurationMs:              persistenceDurationMs,
+		ctx:                                ctx,
 	}, nil
 }
 
 // CountGlobalLockAcquisitionDuration counts the duration of the global lock acquisition
 func (metrics *StoreMetrics) CountGlobalLockAcquisitionDuration(duration time.Duration) {
-	metrics.globalLockAcquisitionDuration.Record(metrics.ctx, duration.Microseconds())
+	metrics.globalLockAcquisitionDurationMicro.Record(metrics.ctx, duration.Microseconds())
+	metrics.globalLockAcquisitionDurationMs.Record(metrics.ctx, duration.Milliseconds())
 }
 
 // CountPersistenceDuration counts the duration of a store persistence operation
 func (metrics *StoreMetrics) CountPersistenceDuration(duration time.Duration) {
-	metrics.persistenceDuration.Record(metrics.ctx, duration.Microseconds())
+	metrics.persistenceDurationMicro.Record(metrics.ctx, duration.Microseconds())
+	metrics.persistenceDurationMs.Record(metrics.ctx, duration.Milliseconds())
 }