From 52ea2e84e9aa3c03fe43c5098ab50c94ff2e0818 Mon Sep 17 00:00:00 2001 From: Pascal Fischer <32096965+pascal-fischer@users.noreply.github.com> Date: Tue, 19 Nov 2024 00:04:50 +0100 Subject: [PATCH] [management] Add transaction metrics and exclude getAccount time from peers update (#2904) --- management/server/peer.go | 11 ++++++----- management/server/sql_store.go | 11 ++++++++++- management/server/telemetry/store_metrics.go | 12 ++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/management/server/peer.go b/management/server/peer.go index 1405dead8..8c45e45c9 100644 --- a/management/server/peer.go +++ b/management/server/peer.go @@ -988,6 +988,12 @@ func (am *DefaultAccountManager) GetPeer(ctx context.Context, accountID, peerID, // updateAccountPeers updates all peers that belong to an account. // Should be called when changes have to be synced to peers. func (am *DefaultAccountManager) updateAccountPeers(ctx context.Context, accountID string) { + account, err := am.requestBuffer.GetAccountWithBackpressure(ctx, accountID) + if err != nil { + log.WithContext(ctx).Errorf("failed to send out updates to peers: %v", err) + return + } + start := time.Now() defer func() { if am.metrics != nil { @@ -995,11 +1001,6 @@ func (am *DefaultAccountManager) updateAccountPeers(ctx context.Context, account } }() - account, err := am.requestBuffer.GetAccountWithBackpressure(ctx, accountID) - if err != nil { - log.WithContext(ctx).Errorf("failed to send out updates to peers: %v", err) - return - } peers := account.GetPeers() approvedPeersMap, err := am.GetValidatedPeers(account) diff --git a/management/server/sql_store.go b/management/server/sql_store.go index 0ebda6440..278f5443d 100644 --- a/management/server/sql_store.go +++ b/management/server/sql_store.go @@ -1123,6 +1123,7 @@ func (s *SqlStore) IncrementNetworkSerial(ctx context.Context, lockStrength Lock } func (s *SqlStore) ExecuteInTransaction(ctx context.Context, operation func(store Store) error) error { + startTime := time.Now() tx := s.db.Begin() if tx.Error != nil { return tx.Error @@ -1133,7 +1134,15 @@ func (s *SqlStore) ExecuteInTransaction(ctx context.Context, operation func(stor tx.Rollback() return err } - return tx.Commit().Error + + err = tx.Commit().Error + + log.WithContext(ctx).Tracef("transaction took %v", time.Since(startTime)) + if s.metrics != nil { + s.metrics.StoreMetrics().CountTransactionDuration(time.Since(startTime)) + } + + return err } func (s *SqlStore) withTx(tx *gorm.DB) Store { diff --git a/management/server/telemetry/store_metrics.go b/management/server/telemetry/store_metrics.go index b038c3d36..bb3745b5a 100644 --- a/management/server/telemetry/store_metrics.go +++ b/management/server/telemetry/store_metrics.go @@ -13,6 +13,7 @@ type StoreMetrics struct { globalLockAcquisitionDurationMs metric.Int64Histogram persistenceDurationMicro metric.Int64Histogram persistenceDurationMs metric.Int64Histogram + transactionDurationMs metric.Int64Histogram ctx context.Context } @@ -40,11 +41,17 @@ func NewStoreMetrics(ctx context.Context, meter metric.Meter) (*StoreMetrics, er return nil, err } + transactionDurationMs, err := meter.Int64Histogram("management.store.transaction.duration.ms") + if err != nil { + return nil, err + } + return &StoreMetrics{ globalLockAcquisitionDurationMicro: globalLockAcquisitionDurationMicro, globalLockAcquisitionDurationMs: globalLockAcquisitionDurationMs, persistenceDurationMicro: persistenceDurationMicro, persistenceDurationMs: persistenceDurationMs, + transactionDurationMs: transactionDurationMs, ctx: ctx, }, nil } @@ -60,3 +67,8 @@ func (metrics *StoreMetrics) CountPersistenceDuration(duration time.Duration) { metrics.persistenceDurationMicro.Record(metrics.ctx, duration.Microseconds()) metrics.persistenceDurationMs.Record(metrics.ctx, duration.Milliseconds()) } + +// CountTransactionDuration counts the duration of a store persistence operation +func (metrics *StoreMetrics) CountTransactionDuration(duration time.Duration) { + metrics.transactionDurationMs.Record(metrics.ctx, duration.Milliseconds()) +}