extern/netbird
1
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2025-06-25 20:22:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix parameter limit issue for Postgres store (#2261)

Browse Source 
Added CreateBatchSize for both SQL stores and updated tests to test large accounts with Postgres, too. Increased the account peer size to 6K.
This commit is contained in:
Maycon Santos 2024-07-12 09:28:53 +02:00 committed by GitHub
parent 1cc341a268
commit 58fbc1249c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 25 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
management/server/sql_store.go
View File

@ -662,11 +662,7 @@ func NewSqliteStore(ctx context.Context, dataDir string, metrics telemetry.AppMe
} }
file := filepath.Join(dataDir, storeStr) file := filepath.Join(dataDir, storeStr)
db, err := gorm.Open(sqlite.Open(file), &gorm.Config{ db, err := gorm.Open(sqlite.Open(file), getGormConfig())
Logger: logger.Default.LogMode(logger.Silent),
CreateBatchSize: 400,
PrepareStmt: true,
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -676,10 +672,7 @@ func NewSqliteStore(ctx context.Context, dataDir string, metrics telemetry.AppMe
// NewPostgresqlStore creates a new Postgres store. // NewPostgresqlStore creates a new Postgres store.
func NewPostgresqlStore(ctx context.Context, dsn string, metrics telemetry.AppMetrics) (*SqlStore, error) { func NewPostgresqlStore(ctx context.Context, dsn string, metrics telemetry.AppMetrics) (*SqlStore, error) {
db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{ db, err := gorm.Open(postgres.Open(dsn), getGormConfig())
Logger: logger.Default.LogMode(logger.Silent),
PrepareStmt: true,
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -687,6 +680,14 @@ func NewPostgresqlStore(ctx context.Context, dsn string, metrics telemetry.AppMe
return NewSqlStore(ctx, db, PostgresStoreEngine, metrics) return NewSqlStore(ctx, db, PostgresStoreEngine, metrics)
} }
func getGormConfig() *gorm.Config {
return &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
CreateBatchSize: 400,
PrepareStmt: true,
}
}
// newPostgresStore initializes a new Postgres store. // newPostgresStore initializes a new Postgres store.
func newPostgresStore(ctx context.Context, metrics telemetry.AppMetrics) (Store, error) { func newPostgresStore(ctx context.Context, metrics telemetry.AppMetrics) (Store, error) {
dsn, ok := os.LookupEnv(postgresDsnEnv) dsn, ok := os.LookupEnv(postgresDsnEnv)

19
management/server/sql_store_test.go
View File

@ -41,11 +41,22 @@ func TestSqlite_NewStore(t *testing.T) {
} }
func TestSqlite_SaveAccount_Large(t *testing.T) { func TestSqlite_SaveAccount_Large(t *testing.T) {
if runtime.GOOS == "windows" { if runtime.GOOS != "linux" && os.Getenv("CI") == "true" || runtime.GOOS == "windows" {
t.Skip("The SQLite store is not properly supported by Windows yet") t.Skip("skip large test on non-linux OS due to environment restrictions")
}
t.Run("SQLite", func(t *testing.T) {
store := newSqliteStore(t)
runLargeTest(t, store)
})
// create store outside to have a better time counter for the test
store := newPostgresqlStore(t)
t.Run("PostgreSQL", func(t *testing.T) {
runLargeTest(t, store)
})
} }
store := newSqliteStore(t) func runLargeTest(t *testing.T, store Store) {
t.Helper()
account := newAccountWithId(context.Background(), "account_id", "testuser", "") account := newAccountWithId(context.Background(), "account_id", "testuser", "")
groupALL, err := account.GetGroupAll() groupALL, err := account.GetGroupAll()
@ -54,7 +65,7 @@ func TestSqlite_SaveAccount_Large(t *testing.T) {
} }
setupKey := GenerateDefaultSetupKey() setupKey := GenerateDefaultSetupKey()
account.SetupKeys[setupKey.Key] = setupKey account.SetupKeys[setupKey.Key] = setupKey
const numPerAccount = 2000 const numPerAccount = 6000
for n := 0; n < numPerAccount; n++ { for n := 0; n < numPerAccount; n++ {
netIP := randomIPv4() netIP := randomIPv4()
peerID := fmt.Sprintf("%s-peer-%d", account.Id, n) peerID := fmt.Sprintf("%s-peer-%d", account.Id, n)