extern/netbird
1
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2025-08-15 17:52:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

update expected benchmark timings

Browse Source
This commit is contained in:
Pascal Fischer
2024-11-29 16:21:10 +01:00
parent 39f8eb72d0
commit 6e2471d623
2 changed files with 114 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

282
management/server/http/setupkeys_handler_benchmark_test.go
View File

@ -17,27 +17,28 @@ import (
"github.com/netbirdio/netbird/management/server/http/api" "github.com/netbirdio/netbird/management/server/http/api"
) )
// Map to store peers, groups, users, and setupKeys by name
var benchCases = map[string]BenchmarkCase{
"Setup Keys - XS": {Peers: 10000, Groups: 10000, Users: 10000, SetupKeys: 5},
"Setup Keys - S": {Peers: 5, Groups: 5, Users: 5, SetupKeys: 100},
"Setup Keys - M": {Peers: 100, Groups: 20, Users: 20, SetupKeys: 1000},
"Setup Keys - L": {Peers: 5, Groups: 5, Users: 5, SetupKeys: 5000},
"Peers - L": {Peers: 10000, Groups: 5, Users: 5, SetupKeys: 5000},
"Groups - L": {Peers: 5, Groups: 10000, Users: 5, SetupKeys: 5000},
"Users - L": {Peers: 5, Groups: 5, Users: 10000, SetupKeys: 5000},
"Setup Keys - XL": {Peers: 500, Groups: 50, Users: 100, SetupKeys: 25000},
}
func BenchmarkCreateSetupKey(b *testing.B) { func BenchmarkCreateSetupKey(b *testing.B) {
benchCases := []struct { var expectedMetrics = map[string]PerformanceMetrics{
name string "Setup Keys - XS": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
peers int "Setup Keys - S": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
groups int "Setup Keys - M": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
users int "Setup Keys - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
setupKeys int "Peers - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
// We need different expectations for CI/CD and local runs because of the different performance characteristics "Groups - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
minMsPerOpLocal float64 "Users - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
maxMsPerOpLocal float64 "Setup Keys - XL": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
minMsPerOpCICD float64
maxMsPerOpCICD float64
}{
{"Setup Keys - XS", 10000, 10000, 10000, 5, 0.5, 2, 1, 12},
{"Setup Keys - S", 5, 5, 5, 100, 0.5, 2, 1, 12},
{"Setup Keys - M", 100, 20, 20, 1000, 0.5, 2, 1, 12},
{"Setup Keys - L", 5, 5, 5, 5000, 0.5, 2, 1, 12},
{"Peers - L", 10000, 5, 5, 5000, 0.5, 2, 1, 12},
{"Groups - L", 5, 10000, 5, 5000, 0.5, 2, 1, 12},
{"Users - L", 5, 5, 10000, 5000, 0.5, 2, 1, 12},
{"Setup Keys - XL", 500, 50, 100, 25000, 0.5, 2, 1, 12},
} }
log.SetOutput(io.Discard) log.SetOutput(io.Discard)
@ -45,10 +46,10 @@ func BenchmarkCreateSetupKey(b *testing.B) {
recorder := httptest.NewRecorder() recorder := httptest.NewRecorder()
for _, bc := range benchCases { for name, bc := range benchCases {
b.Run(bc.name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
apiHandler, am, _ := buildApiBlackBoxWithDBState(b, "testdata/setup_keys.sql", nil) apiHandler, am, _ := buildApiBlackBoxWithDBState(b, "testdata/setup_keys.sql", nil)
populateTestData(b, am.(*server.DefaultAccountManager), bc.peers, bc.groups, bc.users, bc.setupKeys) populateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
b.ResetTimer() b.ResetTimer()
start := time.Now() start := time.Now()
@ -69,49 +70,21 @@ func BenchmarkCreateSetupKey(b *testing.B) {
apiHandler.ServeHTTP(recorder, req) apiHandler.ServeHTTP(recorder, req)
} }
duration := time.Since(start) evaluateBenchmarkResults(b, name, time.Since(start), expectedMetrics[name])
msPerOp := float64(duration.Nanoseconds()) / float64(b.N) / 1e6
b.ReportMetric(msPerOp, "ms/op")
minExpected := bc.minMsPerOpLocal
maxExpected := bc.maxMsPerOpLocal
if os.Getenv("CI") == "true" {
minExpected = bc.minMsPerOpCICD
maxExpected = bc.maxMsPerOpCICD
}
if msPerOp < minExpected {
b.Fatalf("Benchmark %s failed: too fast (%.2f ms/op, minimum %.2f ms/op)", bc.name, msPerOp, minExpected)
}
if msPerOp > maxExpected {
b.Fatalf("Benchmark %s failed: too slow (%.2f ms/op, maximum %.2f ms/op)", bc.name, msPerOp, maxExpected)
}
}) })
} }
} }
func BenchmarkUpdateSetupKey(b *testing.B) { func BenchmarkUpdateSetupKey(b *testing.B) {
benchCases := []struct { var expectedMetrics = map[string]PerformanceMetrics{
name string "Setup Keys - XS": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 3, MinMsPerOpCICD: 3, MaxMsPerOpCICD: 15},
peers int "Setup Keys - S": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 3, MinMsPerOpCICD: 3, MaxMsPerOpCICD: 15},
groups int "Setup Keys - M": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 3, MinMsPerOpCICD: 3, MaxMsPerOpCICD: 15},
users int "Setup Keys - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 3, MinMsPerOpCICD: 3, MaxMsPerOpCICD: 15},
setupKeys int "Peers - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 3, MinMsPerOpCICD: 3, MaxMsPerOpCICD: 15},
// We need different expectations for CI/CD and local runs because of the different performance characteristics "Groups - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 3, MinMsPerOpCICD: 3, MaxMsPerOpCICD: 15},
minMsPerOpLocal float64 "Users - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 3, MinMsPerOpCICD: 3, MaxMsPerOpCICD: 15},
maxMsPerOpLocal float64 "Setup Keys - XL": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 3, MinMsPerOpCICD: 3, MaxMsPerOpCICD: 15},
minMsPerOpCICD float64
maxMsPerOpCICD float64
}{
{"Setup Keys - XS", 10000, 10000, 10000, 5, 0.5, 3, 2, 15},
{"Setup Keys - S", 5, 5, 5, 100, 0.5, 3, 3, 15},
{"Setup Keys - M", 100, 20, 20, 1000, 0.5, 3, 3, 15},
{"Setup Keys - L", 5, 5, 5, 5000, 0.5, 3, 3, 15},
{"Peers - L", 10000, 5, 5, 5000, 0.5, 3, 3, 15},
{"Groups - L", 5, 10000, 5, 5000, 0.5, 3, 3, 15},
{"Users - L", 5, 5, 10000, 5000, 0.5, 3, 3, 15},
{"Setup Keys - XL", 500, 50, 100, 25000, 0.5, 3, 3, 15},
} }
log.SetOutput(io.Discard) log.SetOutput(io.Discard)
@ -119,10 +92,10 @@ func BenchmarkUpdateSetupKey(b *testing.B) {
recorder := httptest.NewRecorder() recorder := httptest.NewRecorder()
for _, bc := range benchCases { for name, bc := range benchCases {
b.Run(bc.name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
apiHandler, am, _ := buildApiBlackBoxWithDBState(b, "testdata/setup_keys.sql", nil) apiHandler, am, _ := buildApiBlackBoxWithDBState(b, "testdata/setup_keys.sql", nil)
populateTestData(b, am.(*server.DefaultAccountManager), bc.peers, bc.groups, bc.users, bc.setupKeys) populateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
b.ResetTimer() b.ResetTimer()
start := time.Now() start := time.Now()
@ -144,49 +117,21 @@ func BenchmarkUpdateSetupKey(b *testing.B) {
apiHandler.ServeHTTP(recorder, req) apiHandler.ServeHTTP(recorder, req)
} }
duration := time.Since(start) evaluateBenchmarkResults(b, name, time.Since(start), expectedMetrics[name])
msPerOp := float64(duration.Nanoseconds()) / float64(b.N) / 1e6
b.ReportMetric(msPerOp, "ms/op")
minExpected := bc.minMsPerOpLocal
maxExpected := bc.maxMsPerOpLocal
if os.Getenv("CI") == "true" {
minExpected = bc.minMsPerOpCICD
maxExpected = bc.maxMsPerOpCICD
}
if msPerOp < minExpected {
b.Fatalf("Benchmark %s failed: too fast (%.2f ms/op, minimum %.2f ms/op)", bc.name, msPerOp, minExpected)
}
if msPerOp > maxExpected {
b.Fatalf("Benchmark %s failed: too slow (%.2f ms/op, maximum %.2f ms/op)", bc.name, msPerOp, maxExpected)
}
}) })
} }
} }
func BenchmarkGetOneSetupKey(b *testing.B) { func BenchmarkGetOneSetupKey(b *testing.B) {
benchCases := []struct { var expectedMetrics = map[string]PerformanceMetrics{
name string "Setup Keys - XS": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
peers int "Setup Keys - S": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
groups int "Setup Keys - M": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
users int "Setup Keys - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
setupKeys int "Peers - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
// We need different expectations for CI/CD and local runs because of the different performance characteristics "Groups - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
minMsPerOpLocal float64 "Users - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
maxMsPerOpLocal float64 "Setup Keys - XL": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
minMsPerOpCICD float64
maxMsPerOpCICD float64
}{
{"Setup Keys - XS", 10000, 10000, 10000, 5, 0.5, 2, 1, 12},
{"Setup Keys - S", 5, 5, 5, 100, 0.5, 2, 1, 12},
{"Setup Keys - M", 100, 20, 20, 1000, 0.5, 2, 1, 12},
{"Setup Keys - L", 5, 5, 5, 5000, 0.5, 2, 1, 12},
{"Peers - L", 10000, 5, 5, 5000, 0.5, 2, 1, 12},
{"Groups - L", 5, 10000, 5, 5000, 0.5, 2, 1, 12},
{"Users - L", 5, 5, 10000, 5000, 0.5, 2, 1, 12},
{"Setup Keys - XL", 500, 50, 100, 25000, 0.5, 2, 1, 12},
} }
log.SetOutput(io.Discard) log.SetOutput(io.Discard)
@ -194,10 +139,10 @@ func BenchmarkGetOneSetupKey(b *testing.B) {
recorder := httptest.NewRecorder() recorder := httptest.NewRecorder()
for _, bc := range benchCases { for name, bc := range benchCases {
b.Run(bc.name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
apiHandler, am, _ := buildApiBlackBoxWithDBState(b, "testdata/setup_keys.sql", nil) apiHandler, am, _ := buildApiBlackBoxWithDBState(b, "testdata/setup_keys.sql", nil)
populateTestData(b, am.(*server.DefaultAccountManager), bc.peers, bc.groups, bc.users, bc.setupKeys) populateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
b.ResetTimer() b.ResetTimer()
start := time.Now() start := time.Now()
@ -206,49 +151,21 @@ func BenchmarkGetOneSetupKey(b *testing.B) {
apiHandler.ServeHTTP(recorder, req) apiHandler.ServeHTTP(recorder, req)
} }
duration := time.Since(start) evaluateBenchmarkResults(b, name, time.Since(start), expectedMetrics[name])
msPerOp := float64(duration.Nanoseconds()) / float64(b.N) / 1e6
b.ReportMetric(msPerOp, "ms/op")
minExpected := bc.minMsPerOpLocal
maxExpected := bc.maxMsPerOpLocal
if os.Getenv("CI") == "true" {
minExpected = bc.minMsPerOpCICD
maxExpected = bc.maxMsPerOpCICD
}
if msPerOp < minExpected {
b.Fatalf("Benchmark %s failed: too fast (%.2f ms/op, minimum %.2f ms/op)", bc.name, msPerOp, minExpected)
}
if msPerOp > maxExpected {
b.Fatalf("Benchmark %s failed: too slow (%.2f ms/op, maximum %.2f ms/op)", bc.name, msPerOp, maxExpected)
}
}) })
} }
} }
func BenchmarkGetAllSetupKeys(b *testing.B) { func BenchmarkGetAllSetupKeys(b *testing.B) {
benchCases := []struct { var expectedMetrics = map[string]PerformanceMetrics{
name string "Setup Keys - XS": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 10},
peers int "Setup Keys - S": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
groups int "Setup Keys - M": {MinMsPerOpLocal: 5, MaxMsPerOpLocal: 10, MinMsPerOpCICD: 5, MaxMsPerOpCICD: 20},
users int "Setup Keys - L": {MinMsPerOpLocal: 30, MaxMsPerOpLocal: 50, MinMsPerOpCICD: 50, MaxMsPerOpCICD: 100},
setupKeys int "Peers - L": {MinMsPerOpLocal: 30, MaxMsPerOpLocal: 50, MinMsPerOpCICD: 50, MaxMsPerOpCICD: 100},
// We need different expectations for CI/CD and local runs because of the different performance characteristics "Groups - L": {MinMsPerOpLocal: 30, MaxMsPerOpLocal: 50, MinMsPerOpCICD: 50, MaxMsPerOpCICD: 100},
minMsPerOpLocal float64 "Users - L": {MinMsPerOpLocal: 30, MaxMsPerOpLocal: 50, MinMsPerOpCICD: 50, MaxMsPerOpCICD: 100},
maxMsPerOpLocal float64 "Setup Keys - XL": {MinMsPerOpLocal: 140, MaxMsPerOpLocal: 220, MinMsPerOpCICD: 300, MaxMsPerOpCICD: 500},
minMsPerOpCICD float64
maxMsPerOpCICD float64
}{
{"Setup Keys - XS", 10000, 10000, 10000, 5, 0.5, 2, 1, 10},
{"Setup Keys - S", 5, 5, 5, 10, 0.5, 2, 1, 12},
{"Setup Keys - M", 100, 20, 20, 1000, 5, 10, 5, 20},
{"Setup Keys - L", 5, 5, 5, 5000, 30, 50, 50, 100},
{"Peers - L", 10000, 5, 5, 5000, 30, 50, 50, 100},
{"Groups - L", 5, 10000, 5, 5000, 30, 50, 50, 100},
{"Users - L", 5, 5, 10000, 5000, 30, 50, 50, 100},
{"Setup Keys - XL", 500, 50, 100, 25000, 140, 220, 300, 500},
} }
log.SetOutput(io.Discard) log.SetOutput(io.Discard)
@ -256,10 +173,10 @@ func BenchmarkGetAllSetupKeys(b *testing.B) {
recorder := httptest.NewRecorder() recorder := httptest.NewRecorder()
for _, bc := range benchCases { for name, bc := range benchCases {
b.Run(bc.name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
apiHandler, am, _ := buildApiBlackBoxWithDBState(b, "testdata/setup_keys.sql", nil) apiHandler, am, _ := buildApiBlackBoxWithDBState(b, "testdata/setup_keys.sql", nil)
populateTestData(b, am.(*server.DefaultAccountManager), bc.peers, bc.groups, bc.users, bc.setupKeys) populateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
b.ResetTimer() b.ResetTimer()
start := time.Now() start := time.Now()
@ -268,49 +185,21 @@ func BenchmarkGetAllSetupKeys(b *testing.B) {
apiHandler.ServeHTTP(recorder, req) apiHandler.ServeHTTP(recorder, req)
} }
duration := time.Since(start) evaluateBenchmarkResults(b, name, time.Since(start), expectedMetrics[name])
msPerOp := float64(duration.Nanoseconds()) / float64(b.N) / 1e6
b.ReportMetric(msPerOp, "ms/op")
minExpected := bc.minMsPerOpLocal
maxExpected := bc.maxMsPerOpLocal
if os.Getenv("CI") == "true" {
minExpected = bc.minMsPerOpCICD
maxExpected = bc.maxMsPerOpCICD
}
if msPerOp < minExpected {
b.Fatalf("Benchmark %s failed: too fast (%.2f ms/op, minimum %.2f ms/op)", bc.name, msPerOp, minExpected)
}
if msPerOp > maxExpected {
b.Fatalf("Benchmark %s failed: too slow (%.2f ms/op, maximum %.2f ms/op)", bc.name, msPerOp, maxExpected)
}
}) })
} }
} }
func BenchmarkDeleteSetupKey(b *testing.B) { func BenchmarkDeleteSetupKey(b *testing.B) {
benchCases := []struct { var expectedMetrics = map[string]PerformanceMetrics{
name string "Setup Keys - XS": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
peers int "Setup Keys - S": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
groups int "Setup Keys - M": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
users int "Setup Keys - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
setupKeys int "Peers - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
// We need different expectations for CI/CD and local runs because of the different performance characteristics "Groups - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
minMsPerOpLocal float64 "Users - L": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
maxMsPerOpLocal float64 "Setup Keys - XL": {MinMsPerOpLocal: 0.5, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 12},
minMsPerOpCICD float64
maxMsPerOpCICD float64
}{
{"Setup Keys - XS", 10000, 10000, 10000, 5, 0.5, 2, 1, 12},
{"Setup Keys - S", 5, 5, 5, 100, 0.5, 2, 1, 12},
{"Setup Keys - M", 100, 20, 20, 1000, 0.5, 2, 1, 12},
{"Setup Keys - L", 5, 5, 5, 5000, 0.5, 2, 1, 12},
{"Peers - L", 10000, 5, 5, 5000, 0.5, 2, 1, 12},
{"Groups - L", 5, 10000, 5, 5000, 0.5, 2, 1, 12},
{"Users - L", 5, 5, 10000, 5000, 0.5, 2, 1, 12},
{"Setup Keys - XL", 500, 50, 100, 25000, 0.5, 2, 1, 12},
} }
log.SetOutput(io.Discard) log.SetOutput(io.Discard)
@ -318,10 +207,10 @@ func BenchmarkDeleteSetupKey(b *testing.B) {
recorder := httptest.NewRecorder() recorder := httptest.NewRecorder()
for _, bc := range benchCases { for name, bc := range benchCases {
b.Run(bc.name, func(b *testing.B) { b.Run(name, func(b *testing.B) {
apiHandler, am, _ := buildApiBlackBoxWithDBState(b, "testdata/setup_keys.sql", nil) apiHandler, am, _ := buildApiBlackBoxWithDBState(b, "testdata/setup_keys.sql", nil)
populateTestData(b, am.(*server.DefaultAccountManager), bc.peers, bc.groups, bc.users, bc.setupKeys) populateTestData(b, am.(*server.DefaultAccountManager), bc.Peers, bc.Groups, bc.Users, bc.SetupKeys)
b.ResetTimer() b.ResetTimer()
start := time.Now() start := time.Now()
@ -331,24 +220,7 @@ func BenchmarkDeleteSetupKey(b *testing.B) {
apiHandler.ServeHTTP(recorder, req) apiHandler.ServeHTTP(recorder, req)
} }
duration := time.Since(start) evaluateBenchmarkResults(b, name, time.Since(start), expectedMetrics[name])
msPerOp := float64(duration.Nanoseconds()) / float64(b.N) / 1e6
b.ReportMetric(msPerOp, "ms/op")
minExpected := bc.minMsPerOpLocal
maxExpected := bc.maxMsPerOpLocal
if os.Getenv("CI") == "true" {
minExpected = bc.minMsPerOpCICD
maxExpected = bc.maxMsPerOpCICD
}
if msPerOp < minExpected {
b.Fatalf("Benchmark %s failed: too fast (%.2f ms/op, minimum %.2f ms/op)", bc.name, msPerOp, minExpected)
}
if msPerOp > maxExpected {
b.Fatalf("Benchmark %s failed: too slow (%.2f ms/op, maximum %.2f ms/op)", bc.name, msPerOp, maxExpected)
}
}) })
} }
} }

37
management/server/http/tools_test.go
View File

@ -8,6 +8,7 @@ import (
"net" "net"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os"
"strconv" "strconv"
"testing" "testing"
"time" "time"
@ -33,6 +34,22 @@ type TB interface {
TempDir() string TempDir() string
} }
// BenchmarkCase defines a single benchmark test case
type BenchmarkCase struct {
Peers int
Groups int
Users int
SetupKeys int
}
// PerformanceMetrics holds the performance expectations
type PerformanceMetrics struct {
MinMsPerOpLocal float64
MaxMsPerOpLocal float64
MinMsPerOpCICD float64
MaxMsPerOpCICD float64
}
func buildApiBlackBoxWithDBState(t TB, sqlFile string, expectedPeerUpdate *server.UpdateMessage) (http.Handler, server.AccountManager, chan struct{}) { func buildApiBlackBoxWithDBState(t TB, sqlFile string, expectedPeerUpdate *server.UpdateMessage) (http.Handler, server.AccountManager, chan struct{}) {
store, cleanup, err := server.NewTestStoreFromSQL(context.Background(), sqlFile, t.TempDir()) store, cleanup, err := server.NewTestStoreFromSQL(context.Background(), sqlFile, t.TempDir())
if err != nil { if err != nil {
@ -227,3 +244,23 @@ func populateTestData(b *testing.B, am *server.DefaultAccountManager, peers, gro
} }
} }
func evaluateBenchmarkResults(b *testing.B, name string, duration time.Duration, perfMetrics PerformanceMetrics) {
msPerOp := float64(duration.Nanoseconds()) / float64(b.N) / 1e6
b.ReportMetric(msPerOp, "ms/op")
minExpected := perfMetrics.MinMsPerOpLocal
maxExpected := perfMetrics.MaxMsPerOpLocal
if os.Getenv("CI") == "true" {
minExpected = perfMetrics.MinMsPerOpCICD
maxExpected = perfMetrics.MaxMsPerOpCICD
}
if msPerOp < minExpected {
b.Fatalf("Benchmark %s failed: too fast (%.2f ms/op, minimum %.2f ms/op)", name, msPerOp, minExpected)
}
if msPerOp > maxExpected {
b.Fatalf("Benchmark %s failed: too slow (%.2f ms/op, maximum %.2f ms/op)", name, msPerOp, maxExpected)
}
}