diff --git a/go.mod b/go.mod index 0f779e35d..c86acdf26 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,6 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/libdns/route53 v1.5.0 github.com/libp2p/go-netroute v0.2.1 - github.com/mattn/go-sqlite3 v1.14.22 github.com/mdlayher/socket v0.5.1 github.com/miekg/dns v1.1.59 github.com/mitchellh/hashstructure/v2 v2.0.2 @@ -195,6 +194,7 @@ require ( github.com/libdns/libdns v0.2.2 // indirect github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/mdlayher/genetlink v1.3.2 // indirect github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect github.com/mholt/acmez/v2 v2.0.1 // indirect diff --git a/management/server/activity/event.go b/management/server/activity/event.go index 0e819c3a7..5bcc67fbc 100644 --- a/management/server/activity/event.go +++ b/management/server/activity/event.go @@ -19,22 +19,22 @@ type Event struct { // Timestamp of the event Timestamp time.Time // Activity that was performed during the event - Activity ActivityDescriber + Activity Activity `gorm:"type:integer"` // ID of the event (can be empty, meaning that it wasn't yet generated) - ID uint64 + ID uint64 `gorm:"primaryKey;autoIncrement"` // InitiatorID is the ID of an object that initiated the event (e.g., a user) InitiatorID string // InitiatorName is the name of an object that initiated the event. - InitiatorName string + InitiatorName string `gorm:"-"` // InitiatorEmail is the email address of an object that initiated the event. - InitiatorEmail string + InitiatorEmail string `gorm:"-"` // TargetID is the ID of an object that was effected by the event (e.g., a peer) TargetID string // AccountID is the ID of an account where the event happened AccountID string // Meta of the event, e.g. deleted peer information like name, IP, etc - Meta map[string]any + Meta map[string]any `gorm:"serializer:json"` } // Copy the event @@ -57,3 +57,10 @@ func (e *Event) Copy() *Event { Meta: meta, } } + +type DeletedUser struct { + ID string `gorm:"primaryKey"` + Email string `gorm:"not null"` + Name string + EncAlgo string `gorm:"not null"` +} diff --git a/management/server/activity/sqlite/migration.go b/management/server/activity/sqlite/migration.go index 28c5b3020..6da7893a0 100644 --- a/management/server/activity/sqlite/migration.go +++ b/management/server/activity/sqlite/migration.go @@ -2,156 +2,180 @@ package sqlite import ( "context" - "database/sql" "fmt" log "github.com/sirupsen/logrus" + "gorm.io/gorm" + + "github.com/netbirdio/netbird/management/server/activity" + "github.com/netbirdio/netbird/management/server/migration" ) -func migrate(ctx context.Context, crypt *FieldEncrypt, db *sql.DB) error { - if _, err := db.Exec(createTableQuery); err != nil { - return err - } +func migrate(ctx context.Context, crypt *FieldEncrypt, db *gorm.DB) error { + migrations := getMigrations(ctx, crypt) - if _, err := db.Exec(creatTableDeletedUsersQuery); err != nil { - return err - } - - if err := updateDeletedUsersTable(ctx, db); err != nil { - return fmt.Errorf("failed to update deleted_users table: %v", err) - } - - return migrateLegacyEncryptedUsersToGCM(ctx, crypt, db) -} - -// updateDeletedUsersTable checks and updates the deleted_users table schema to ensure required columns exist. -func updateDeletedUsersTable(ctx context.Context, db *sql.DB) error { - exists, err := checkColumnExists(db, "deleted_users", "name") - if err != nil { - return err - } - - if !exists { - log.WithContext(ctx).Debug("Adding name column to the deleted_users table") - - _, err = db.Exec(`ALTER TABLE deleted_users ADD COLUMN name TEXT;`) - if err != nil { + for _, m := range migrations { + if err := m(db); err != nil { return err } - - log.WithContext(ctx).Debug("Successfully added name column to the deleted_users table") - } - - exists, err = checkColumnExists(db, "deleted_users", "enc_algo") - if err != nil { - return err - } - - if !exists { - log.WithContext(ctx).Debug("Adding enc_algo column to the deleted_users table") - - _, err = db.Exec(`ALTER TABLE deleted_users ADD COLUMN enc_algo TEXT;`) - if err != nil { - return err - } - - log.WithContext(ctx).Debug("Successfully added enc_algo column to the deleted_users table") } return nil } -// migrateLegacyEncryptedUsersToGCM migrates previously encrypted data using, +type migrationFunc func(*gorm.DB) error + +func getMigrations(ctx context.Context, crypt *FieldEncrypt) []migrationFunc { + return []migrationFunc{ + func(db *gorm.DB) error { + return migration.MigrateNewField[activity.DeletedUser](ctx, db, "name", "") + }, + func(db *gorm.DB) error { + return migration.MigrateNewField[activity.DeletedUser](ctx, db, "enc_algo", "") + }, + func(db *gorm.DB) error { + return migrateLegacyEncryptedUsersToGCM(ctx, db, crypt) + }, + func(db *gorm.DB) error { + return migrateDuplicateDeletedUsers(ctx, db) + }, + } +} + +// migrateLegacyEncryptedUsersToGCM migrates previously encrypted data using // legacy CBC encryption with a static IV to the new GCM encryption method. -func migrateLegacyEncryptedUsersToGCM(ctx context.Context, crypt *FieldEncrypt, db *sql.DB) error { - log.WithContext(ctx).Debug("Migrating CBC encrypted deleted users to GCM") +func migrateLegacyEncryptedUsersToGCM(ctx context.Context, db *gorm.DB, crypt *FieldEncrypt) error { + model := &activity.DeletedUser{} - tx, err := db.Begin() - if err != nil { - return fmt.Errorf("failed to begin transaction: %v", err) + if !db.Migrator().HasTable(model) { + log.WithContext(ctx).Debugf("Table for %T does not exist, no CBC to GCM migration needed", model) + return nil } - defer func() { - _ = tx.Rollback() - }() - rows, err := tx.Query(fmt.Sprintf(`SELECT id, email, name FROM deleted_users where enc_algo IS NULL OR enc_algo != '%s'`, gcmEncAlgo)) + var deletedUsers []activity.DeletedUser + err := db.Model(model).Find(&deletedUsers, "enc_algo IS NULL OR enc_algo != ?", gcmEncAlgo).Error if err != nil { - return fmt.Errorf("failed to execute select query: %v", err) + return fmt.Errorf("failed to query deleted_users: %w", err) } - defer rows.Close() - updateStmt, err := tx.Prepare(`UPDATE deleted_users SET email = ?, name = ?, enc_algo = ? WHERE id = ?`) - if err != nil { - return fmt.Errorf("failed to prepare update statement: %v", err) + if len(deletedUsers) == 0 { + log.WithContext(ctx).Debug("No CBC encrypted deleted users to migrate") + return nil } - defer updateStmt.Close() - if err = processUserRows(ctx, crypt, rows, updateStmt); err != nil { + if err = db.Transaction(func(tx *gorm.DB) error { + for _, user := range deletedUsers { + if err = updateDeletedUserData(tx, user, crypt); err != nil { + return fmt.Errorf("failed to migrate deleted user %s: %w", user.ID, err) + } + } + return nil + }); err != nil { return err } - if err = tx.Commit(); err != nil { - return fmt.Errorf("failed to commit transaction: %v", err) - } - log.WithContext(ctx).Debug("Successfully migrated CBC encrypted deleted users to GCM") + return nil } -// processUserRows processes database rows of user data, decrypts legacy encryption fields, and re-encrypts them using GCM. -func processUserRows(ctx context.Context, crypt *FieldEncrypt, rows *sql.Rows, updateStmt *sql.Stmt) error { - for rows.Next() { - var ( - id, decryptedEmail, decryptedName string - email, name *string - ) +func updateDeletedUserData(transaction *gorm.DB, user activity.DeletedUser, crypt *FieldEncrypt) error { + var err error + var decryptedEmail, decryptedName string - err := rows.Scan(&id, &email, &name) + if user.Email != "" { + decryptedEmail, err = crypt.LegacyDecrypt(user.Email) if err != nil { - return err - } - - if email != nil { - decryptedEmail, err = crypt.LegacyDecrypt(*email) - if err != nil { - log.WithContext(ctx).Warnf("skipping migrating deleted user %s: %v", - id, - fmt.Errorf("failed to decrypt email: %w", err), - ) - continue - } - } - - if name != nil { - decryptedName, err = crypt.LegacyDecrypt(*name) - if err != nil { - log.WithContext(ctx).Warnf("skipping migrating deleted user %s: %v", - id, - fmt.Errorf("failed to decrypt name: %w", err), - ) - continue - } - } - - encryptedEmail, err := crypt.Encrypt(decryptedEmail) - if err != nil { - return fmt.Errorf("failed to encrypt email: %w", err) - } - - encryptedName, err := crypt.Encrypt(decryptedName) - if err != nil { - return fmt.Errorf("failed to encrypt name: %w", err) - } - - _, err = updateStmt.Exec(encryptedEmail, encryptedName, gcmEncAlgo, id) - if err != nil { - return err + return fmt.Errorf("failed to decrypt email: %w", err) } } - if err := rows.Err(); err != nil { + if user.Name != "" { + decryptedName, err = crypt.LegacyDecrypt(user.Name) + if err != nil { + return fmt.Errorf("failed to decrypt name: %w", err) + } + } + + updatedUser := user + updatedUser.EncAlgo = gcmEncAlgo + + updatedUser.Email, err = crypt.Encrypt(decryptedEmail) + if err != nil { + return fmt.Errorf("failed to encrypt email: %w", err) + } + + updatedUser.Name, err = crypt.Encrypt(decryptedName) + if err != nil { + return fmt.Errorf("failed to encrypt name: %w", err) + } + + return transaction.Model(&updatedUser).Omit("id").Updates(updatedUser).Error +} + +// MigrateDuplicateDeletedUsers removes duplicates and ensures the id column is marked as the primary key +func migrateDuplicateDeletedUsers(ctx context.Context, db *gorm.DB) error { + model := &activity.DeletedUser{} + if !db.Migrator().HasTable(model) { + log.WithContext(ctx).Debugf("Table for %T does not exist, no duplicate migration needed", model) + return nil + } + + isPrimaryKey, err := isColumnPrimaryKey[activity.DeletedUser](db, "id") + if err != nil { return err } + if isPrimaryKey { + log.WithContext(ctx).Debug("No duplicate deleted users to migrate") + return nil + } + + if err = db.Transaction(func(tx *gorm.DB) error { + groupById := tx.Model(model).Select("MAX(rowid)").Group("id") + if err = tx.Delete(model, "rowid NOT IN (?)", groupById).Error; err != nil { + return err + } + + if err = tx.Migrator().RenameTable("deleted_users", "deleted_users_old"); err != nil { + return err + } + + if err = tx.Migrator().CreateTable(model); err != nil { + return err + } + + if err = tx.Exec(` + INSERT INTO deleted_users (id, email, name, enc_algo) SELECT id, email, name, enc_algo + FROM deleted_users_old;`).Error; err != nil { + return err + } + + return tx.Migrator().DropTable("deleted_users_old") + }); err != nil { + return err + } + + log.WithContext(ctx).Debug("Successfully migrated duplicate deleted users") + return nil } + +// isColumnPrimaryKey checks if a column is a primary key in the given model +func isColumnPrimaryKey[T any](db *gorm.DB, columnName string) (bool, error) { + var model T + + cols, err := db.Migrator().ColumnTypes(&model) + if err != nil { + return false, err + } + + for _, col := range cols { + if col.Name() == columnName { + isPrimaryKey, _ := col.PrimaryKey() + return isPrimaryKey, nil + } + } + + return false, nil +} diff --git a/management/server/activity/sqlite/migration_test.go b/management/server/activity/sqlite/migration_test.go index a03774fa8..498c976d9 100644 --- a/management/server/activity/sqlite/migration_test.go +++ b/management/server/activity/sqlite/migration_test.go @@ -2,38 +2,39 @@ package sqlite import ( "context" - "database/sql" "path/filepath" "testing" - "time" - - _ "github.com/mattn/go-sqlite3" - "github.com/netbirdio/netbird/management/server/activity" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/netbirdio/netbird/management/server/activity" + "github.com/netbirdio/netbird/management/server/migration" ) -func setupDatabase(t *testing.T) *sql.DB { +const ( + insertDeletedUserQuery = `INSERT INTO deleted_users (id, email, name, enc_algo) VALUES (?, ?, ?, ?)` +) + +func setupDatabase(t *testing.T) *gorm.DB { t.Helper() dbFile := filepath.Join(t.TempDir(), eventSinkDB) - db, err := sql.Open("sqlite3", dbFile) - require.NoError(t, err, "Failed to open database") + db, err := gorm.Open(sqlite.Open(dbFile)) + require.NoError(t, err) + sql, err := db.DB() + require.NoError(t, err) t.Cleanup(func() { - _ = db.Close() + _ = sql.Close() }) - _, err = db.Exec(createTableQuery) - require.NoError(t, err, "Failed to create events table") - - _, err = db.Exec(`CREATE TABLE deleted_users (id TEXT NOT NULL, email TEXT NOT NULL, name TEXT);`) - require.NoError(t, err, "Failed to create deleted_users table") - return db } -func TestMigrate(t *testing.T) { +func TestMigrateLegacyEncryptedUsersToGCM(t *testing.T) { db := setupDatabase(t) key, err := GenerateKey() @@ -42,43 +43,98 @@ func TestMigrate(t *testing.T) { crypt, err := NewFieldEncrypt(key) require.NoError(t, err, "Failed to initialize FieldEncrypt") - legacyEmail := crypt.LegacyEncrypt("testaccount@test.com") - legacyName := crypt.LegacyEncrypt("Test Account") + t.Run("empty table, no migration required", func(t *testing.T) { + require.NoError(t, migrateLegacyEncryptedUsersToGCM(context.Background(), db, crypt)) + assert.False(t, db.Migrator().HasTable("deleted_users")) + }) - _, err = db.Exec(`INSERT INTO events(activity, timestamp, initiator_id, target_id, account_id, meta) VALUES(?, ?, ?, ?, ?, ?)`, - activity.UserDeleted, time.Now(), "initiatorID", "targetID", "accountID", "") - require.NoError(t, err, "Failed to insert event") + require.NoError(t, db.Exec(`CREATE TABLE deleted_users (id TEXT NOT NULL, email TEXT NOT NULL, name TEXT);`).Error) + assert.True(t, db.Migrator().HasTable("deleted_users")) + assert.False(t, db.Migrator().HasColumn("deleted_users", "enc_algo")) - _, err = db.Exec(`INSERT INTO deleted_users(id, email, name) VALUES(?, ?, ?)`, "targetID", legacyEmail, legacyName) - require.NoError(t, err, "Failed to insert legacy encrypted data") + require.NoError(t, migration.MigrateNewField[activity.DeletedUser](context.Background(), db, "enc_algo", "")) + assert.True(t, db.Migrator().HasColumn("deleted_users", "enc_algo")) - colExists, err := checkColumnExists(db, "deleted_users", "enc_algo") - require.NoError(t, err, "Failed to check if enc_algo column exists") - require.False(t, colExists, "enc_algo column should not exist before migration") + t.Run("legacy users migration", func(t *testing.T) { + legacyEmail := crypt.LegacyEncrypt("test.user@test.com") + legacyName := crypt.LegacyEncrypt("Test User") - err = migrate(context.Background(), crypt, db) - require.NoError(t, err, "Migration failed") + require.NoError(t, db.Exec(insertDeletedUserQuery, "user1", legacyEmail, legacyName, "").Error) + require.NoError(t, db.Exec(insertDeletedUserQuery, "user2", legacyEmail, legacyName, "legacy").Error) - colExists, err = checkColumnExists(db, "deleted_users", "enc_algo") - require.NoError(t, err, "Failed to check if enc_algo column exists after migration") - require.True(t, colExists, "enc_algo column should exist after migration") + require.NoError(t, migrateLegacyEncryptedUsersToGCM(context.Background(), db, crypt)) - var encAlgo string - err = db.QueryRow(`SELECT enc_algo FROM deleted_users LIMIT 1`, "").Scan(&encAlgo) - require.NoError(t, err, "Failed to select updated data") - require.Equal(t, gcmEncAlgo, encAlgo, "enc_algo should be set to 'GCM' after migration") + var users []activity.DeletedUser + require.NoError(t, db.Find(&users).Error) + assert.Len(t, users, 2) - store, err := createStore(crypt, db) - require.NoError(t, err, "Failed to create store") + for _, user := range users { + assert.Equal(t, gcmEncAlgo, user.EncAlgo) - events, err := store.Get(context.Background(), "accountID", 0, 1, false) - require.NoError(t, err, "Failed to get events") + decryptedEmail, err := crypt.Decrypt(user.Email) + require.NoError(t, err) + assert.Equal(t, "test.user@test.com", decryptedEmail) - require.Len(t, events, 1, "Should have one event") - require.Equal(t, activity.UserDeleted, events[0].Activity, "activity should match") - require.Equal(t, "initiatorID", events[0].InitiatorID, "initiator id should match") - require.Equal(t, "targetID", events[0].TargetID, "target id should match") - require.Equal(t, "accountID", events[0].AccountID, "account id should match") - require.Equal(t, "testaccount@test.com", events[0].Meta["email"], "email should match") - require.Equal(t, "Test Account", events[0].Meta["username"], "username should match") + decryptedName, err := crypt.Decrypt(user.Name) + require.NoError(t, err) + require.Equal(t, "Test User", decryptedName) + } + }) + + t.Run("users already migrated, no migration", func(t *testing.T) { + encryptedEmail, err := crypt.Encrypt("test.user@test.com") + require.NoError(t, err) + + encryptedName, err := crypt.Encrypt("Test User") + require.NoError(t, err) + + require.NoError(t, db.Exec(insertDeletedUserQuery, "user3", encryptedEmail, encryptedName, gcmEncAlgo).Error) + require.NoError(t, migrateLegacyEncryptedUsersToGCM(context.Background(), db, crypt)) + + var users []activity.DeletedUser + require.NoError(t, db.Find(&users).Error) + assert.Len(t, users, 3) + + for _, user := range users { + assert.Equal(t, gcmEncAlgo, user.EncAlgo) + + decryptedEmail, err := crypt.Decrypt(user.Email) + require.NoError(t, err) + assert.Equal(t, "test.user@test.com", decryptedEmail) + + decryptedName, err := crypt.Decrypt(user.Name) + require.NoError(t, err) + require.Equal(t, "Test User", decryptedName) + } + }) +} + +func TestMigrateDuplicateDeletedUsers(t *testing.T) { + db := setupDatabase(t) + + require.NoError(t, migrateDuplicateDeletedUsers(context.Background(), db)) + assert.False(t, db.Migrator().HasTable("deleted_users")) + + require.NoError(t, db.Exec(`CREATE TABLE deleted_users (id TEXT NOT NULL, email TEXT NOT NULL, name TEXT, enc_algo TEXT NOT NULL);`).Error) + assert.True(t, db.Migrator().HasTable("deleted_users")) + + isPrimaryKey, err := isColumnPrimaryKey[activity.DeletedUser](db, "id") + require.NoError(t, err) + assert.False(t, isPrimaryKey) + + require.NoError(t, db.Exec(insertDeletedUserQuery, "user1", "email1", "name1", "GCM").Error) + require.NoError(t, db.Exec(insertDeletedUserQuery, "user1", "email2", "name2", "GCM").Error) + require.NoError(t, migrateDuplicateDeletedUsers(context.Background(), db)) + + isPrimaryKey, err = isColumnPrimaryKey[activity.DeletedUser](db, "id") + require.NoError(t, err) + assert.True(t, isPrimaryKey) + + var users []activity.DeletedUser + require.NoError(t, db.Find(&users).Error) + assert.Len(t, users, 1) + assert.Equal(t, "user1", users[0].ID) + assert.Equal(t, "email2", users[0].Email) + assert.Equal(t, "name2", users[0].Name) + assert.Equal(t, "GCM", users[0].EncAlgo) } diff --git a/management/server/activity/sqlite/sqlite.go b/management/server/activity/sqlite/sqlite.go index ffb863de9..6d198fca9 100644 --- a/management/server/activity/sqlite/sqlite.go +++ b/management/server/activity/sqlite/sqlite.go @@ -2,75 +2,21 @@ package sqlite import ( "context" - "database/sql" - "encoding/json" "fmt" "path/filepath" - "runtime" - "time" - _ "github.com/mattn/go-sqlite3" log "github.com/sirupsen/logrus" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/logger" "github.com/netbirdio/netbird/management/server/activity" ) const ( // eventSinkDB is the default name of the events database - eventSinkDB = "events.db" - createTableQuery = "CREATE TABLE IF NOT EXISTS events " + - "(id INTEGER PRIMARY KEY AUTOINCREMENT, " + - "activity INTEGER, " + - "timestamp DATETIME, " + - "initiator_id TEXT," + - "account_id TEXT," + - "meta TEXT," + - " target_id TEXT);" - - creatTableDeletedUsersQuery = `CREATE TABLE IF NOT EXISTS deleted_users (id TEXT NOT NULL, email TEXT NOT NULL, name TEXT, enc_algo TEXT NOT NULL);` - - selectDescQuery = `SELECT events.id, activity, timestamp, initiator_id, i.name as "initiator_name", i.email as "initiator_email", target_id, t.name as "target_name", t.email as "target_email", account_id, meta - FROM events - LEFT JOIN ( - SELECT id, MAX(name) as name, MAX(email) as email - FROM deleted_users - GROUP BY id - ) i ON events.initiator_id = i.id - LEFT JOIN ( - SELECT id, MAX(name) as name, MAX(email) as email - FROM deleted_users - GROUP BY id - ) t ON events.target_id = t.id - WHERE account_id = ? - ORDER BY timestamp DESC LIMIT ? OFFSET ?;` - - selectAscQuery = `SELECT events.id, activity, timestamp, initiator_id, i.name as "initiator_name", i.email as "initiator_email", target_id, t.name as "target_name", t.email as "target_email", account_id, meta - FROM events - LEFT JOIN ( - SELECT id, MAX(name) as name, MAX(email) as email - FROM deleted_users - GROUP BY id - ) i ON events.initiator_id = i.id - LEFT JOIN ( - SELECT id, MAX(name) as name, MAX(email) as email - FROM deleted_users - GROUP BY id - ) t ON events.target_id = t.id - WHERE account_id = ? - ORDER BY timestamp ASC LIMIT ? OFFSET ?;` - - insertQuery = "INSERT INTO events(activity, timestamp, initiator_id, target_id, account_id, meta) " + - "VALUES(?, ?, ?, ?, ?, ?)" - - /* - TODO: - The insert should avoid duplicated IDs in the table. So the query should be changes to something like: - `INSERT INTO deleted_users(id, email, name) VALUES(?, ?, ?) ON CONFLICT (id) DO UPDATE SET email = EXCLUDED.email, name = EXCLUDED.name;` - For this to work we have to set the id column as primary key. But this is not possible because the id column is not unique - and some selfhosted deployments might have duplicates already so we need to clean the table first. - */ - - insertDeleteUserQuery = `INSERT INTO deleted_users(id, email, name, enc_algo) VALUES(?, ?, ?, ?)` + eventSinkDB = "events.db" fallbackName = "unknown" fallbackEmail = "unknown@unknown.com" @@ -78,172 +24,158 @@ const ( gcmEncAlgo = "GCM" ) +type eventWithNames struct { + activity.Event + InitiatorName string + InitiatorEmail string + TargetName string + TargetEmail string +} + // Store is the implementation of the activity.Store interface backed by SQLite type Store struct { - db *sql.DB + db *gorm.DB fieldEncrypt *FieldEncrypt - - insertStatement *sql.Stmt - selectAscStatement *sql.Stmt - selectDescStatement *sql.Stmt - deleteUserStmt *sql.Stmt } // NewSQLiteStore creates a new Store with an event table if not exists. func NewSQLiteStore(ctx context.Context, dataDir string, encryptionKey string) (*Store, error) { - dbFile := filepath.Join(dataDir, eventSinkDB) - db, err := sql.Open("sqlite3", dbFile) - if err != nil { - return nil, err - } - db.SetMaxOpenConns(runtime.NumCPU()) - crypt, err := NewFieldEncrypt(encryptionKey) if err != nil { - _ = db.Close() + return nil, err } + dbFile := filepath.Join(dataDir, eventSinkDB) + db, err := gorm.Open(sqlite.Open(dbFile), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + return nil, err + } + + sql, err := db.DB() + if err != nil { + return nil, err + } + sql.SetMaxOpenConns(1) + if err = migrate(ctx, crypt, db); err != nil { - _ = db.Close() return nil, fmt.Errorf("events database migration: %w", err) } - return createStore(crypt, db) + err = db.AutoMigrate(&activity.Event{}, &activity.DeletedUser{}) + if err != nil { + return nil, fmt.Errorf("events auto migrate: %w", err) + } + + return &Store{ + db: db, + fieldEncrypt: crypt, + }, nil } -func (store *Store) processResult(ctx context.Context, result *sql.Rows) ([]*activity.Event, error) { - events := make([]*activity.Event, 0) +func (store *Store) processResult(ctx context.Context, events []*eventWithNames) ([]*activity.Event, error) { + activityEvents := make([]*activity.Event, 0) var cryptErr error - for result.Next() { - var id int64 - var operation activity.Activity - var timestamp time.Time - var initiator string - var initiatorName *string - var initiatorEmail *string - var target string - var targetUserName *string - var targetEmail *string - var account string - var jsonMeta string - err := result.Scan(&id, &operation, ×tamp, &initiator, &initiatorName, &initiatorEmail, &target, &targetUserName, &targetEmail, &account, &jsonMeta) - if err != nil { - return nil, err + + for _, event := range events { + e := event.Event + if e.Meta == nil { + e.Meta = make(map[string]any) } - meta := make(map[string]any) - if jsonMeta != "" { - err = json.Unmarshal([]byte(jsonMeta), &meta) + if event.TargetName != "" { + name, err := store.fieldEncrypt.Decrypt(event.TargetName) if err != nil { - return nil, err - } - } - - if targetUserName != nil { - name, err := store.fieldEncrypt.Decrypt(*targetUserName) - if err != nil { - cryptErr = fmt.Errorf("failed to decrypt username for target id: %s", target) - meta["username"] = fallbackName + cryptErr = fmt.Errorf("failed to decrypt username for target id: %s", event.TargetName) + e.Meta["username"] = fallbackName } else { - meta["username"] = name + e.Meta["username"] = name } } - if targetEmail != nil { - email, err := store.fieldEncrypt.Decrypt(*targetEmail) + if event.TargetEmail != "" { + email, err := store.fieldEncrypt.Decrypt(event.TargetEmail) if err != nil { - cryptErr = fmt.Errorf("failed to decrypt email address for target id: %s", target) - meta["email"] = fallbackEmail + cryptErr = fmt.Errorf("failed to decrypt email address for target id: %s", event.TargetEmail) + e.Meta["email"] = fallbackEmail } else { - meta["email"] = email + e.Meta["email"] = email } } - event := &activity.Event{ - Timestamp: timestamp, - Activity: operation, - ID: uint64(id), - InitiatorID: initiator, - TargetID: target, - AccountID: account, - Meta: meta, - } - - if initiatorName != nil { - name, err := store.fieldEncrypt.Decrypt(*initiatorName) + if event.InitiatorName != "" { + name, err := store.fieldEncrypt.Decrypt(event.InitiatorName) if err != nil { - cryptErr = fmt.Errorf("failed to decrypt username of initiator: %s", initiator) - event.InitiatorName = fallbackName + cryptErr = fmt.Errorf("failed to decrypt username of initiator: %s", event.InitiatorName) + e.InitiatorName = fallbackName } else { - event.InitiatorName = name + e.InitiatorName = name } } - if initiatorEmail != nil { - email, err := store.fieldEncrypt.Decrypt(*initiatorEmail) + if event.InitiatorEmail != "" { + email, err := store.fieldEncrypt.Decrypt(event.InitiatorEmail) if err != nil { - cryptErr = fmt.Errorf("failed to decrypt email address of initiator: %s", initiator) - event.InitiatorEmail = fallbackEmail + cryptErr = fmt.Errorf("failed to decrypt email address of initiator: %s", event.InitiatorEmail) + e.InitiatorEmail = fallbackEmail } else { - event.InitiatorEmail = email + e.InitiatorEmail = email } } - events = append(events, event) + activityEvents = append(activityEvents, &e) } if cryptErr != nil { log.WithContext(ctx).Warnf("%s", cryptErr) } - return events, nil + return activityEvents, nil } // Get returns "limit" number of events from index ordered descending or ascending by a timestamp func (store *Store) Get(ctx context.Context, accountID string, offset, limit int, descending bool) ([]*activity.Event, error) { - stmt := store.selectDescStatement + baseQuery := store.db.Model(&activity.Event{}). + Select(` + events.*, + u.name AS initiator_name, + u.email AS initiator_email, + t.name AS target_name, + t.email AS target_email + `). + Joins(`LEFT JOIN deleted_users u ON u.id = events.initiator_id`). + Joins(`LEFT JOIN deleted_users t ON t.id = events.target_id`) + + orderDir := "DESC" if !descending { - stmt = store.selectAscStatement + orderDir = "ASC" } - result, err := stmt.Query(accountID, limit, offset) + var events []*eventWithNames + err := baseQuery.Order("events.timestamp "+orderDir).Offset(offset).Limit(limit). + Find(&events, "account_id = ?", accountID).Error if err != nil { return nil, err } - defer result.Close() //nolint - return store.processResult(ctx, result) + return store.processResult(ctx, events) } // Save an event in the SQLite events table end encrypt the "email" element in meta map func (store *Store) Save(_ context.Context, event *activity.Event) (*activity.Event, error) { - var jsonMeta string - meta, err := store.saveDeletedUserEmailAndNameInEncrypted(event) - if err != nil { - return nil, err - } - - if meta != nil { - metaBytes, err := json.Marshal(event.Meta) - if err != nil { - return nil, err - } - jsonMeta = string(metaBytes) - } - - result, err := store.insertStatement.Exec(event.Activity, event.Timestamp, event.InitiatorID, event.TargetID, event.AccountID, jsonMeta) - if err != nil { - return nil, err - } - - id, err := result.LastInsertId() - if err != nil { - return nil, err - } - eventCopy := event.Copy() - eventCopy.ID = uint64(id) + meta, err := store.saveDeletedUserEmailAndNameInEncrypted(eventCopy) + if err != nil { + return nil, err + } + eventCopy.Meta = meta + + if err = store.db.Create(eventCopy).Error; err != nil { + return nil, err + } + return eventCopy, nil } @@ -260,16 +192,27 @@ func (store *Store) saveDeletedUserEmailAndNameInEncrypted(event *activity.Event return event.Meta, nil } + deletedUser := activity.DeletedUser{ + ID: event.TargetID, + EncAlgo: gcmEncAlgo, + } + encryptedEmail, err := store.fieldEncrypt.Encrypt(fmt.Sprintf("%s", email)) if err != nil { return nil, err } + deletedUser.Email = encryptedEmail + encryptedName, err := store.fieldEncrypt.Encrypt(fmt.Sprintf("%s", name)) if err != nil { return nil, err } + deletedUser.Name = encryptedName - _, err = store.deleteUserStmt.Exec(event.TargetID, encryptedEmail, encryptedName, gcmEncAlgo) + err = store.db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "id"}}, + DoUpdates: clause.AssignmentColumns([]string{"email", "name"}), + }).Create(deletedUser).Error if err != nil { return nil, err } @@ -285,75 +228,11 @@ func (store *Store) saveDeletedUserEmailAndNameInEncrypted(event *activity.Event // Close the Store func (store *Store) Close(_ context.Context) error { if store.db != nil { - return store.db.Close() + sql, err := store.db.DB() + if err != nil { + return err + } + return sql.Close() } return nil } - -// createStore initializes and returns a new Store instance with prepared SQL statements. -func createStore(crypt *FieldEncrypt, db *sql.DB) (*Store, error) { - insertStmt, err := db.Prepare(insertQuery) - if err != nil { - _ = db.Close() - return nil, err - } - - selectDescStmt, err := db.Prepare(selectDescQuery) - if err != nil { - _ = db.Close() - return nil, err - } - - selectAscStmt, err := db.Prepare(selectAscQuery) - if err != nil { - _ = db.Close() - return nil, err - } - - deleteUserStmt, err := db.Prepare(insertDeleteUserQuery) - if err != nil { - _ = db.Close() - return nil, err - } - - return &Store{ - db: db, - fieldEncrypt: crypt, - insertStatement: insertStmt, - selectDescStatement: selectDescStmt, - selectAscStatement: selectAscStmt, - deleteUserStmt: deleteUserStmt, - }, nil -} - -// checkColumnExists checks if a column exists in a specified table -func checkColumnExists(db *sql.DB, tableName, columnName string) (bool, error) { - query := fmt.Sprintf("PRAGMA table_info(%s);", tableName) - rows, err := db.Query(query) - if err != nil { - return false, fmt.Errorf("failed to query table info: %w", err) - } - defer rows.Close() - - for rows.Next() { - var cid int - var name, ctype string - var notnull, pk int - var dfltValue sql.NullString - - err = rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk) - if err != nil { - return false, fmt.Errorf("failed to scan row: %w", err) - } - - if name == columnName { - return true, nil - } - } - - if err = rows.Err(); err != nil { - return false, err - } - - return false, nil -} diff --git a/management/server/event.go b/management/server/event.go index 6342bfedb..2952edc8c 100644 --- a/management/server/event.go +++ b/management/server/event.go @@ -66,7 +66,7 @@ func (am *DefaultAccountManager) StoreEvent(ctx context.Context, initiatorID, ta go func() { _, err := am.eventStore.Save(ctx, &activity.Event{ Timestamp: time.Now().UTC(), - Activity: activityID, + Activity: activityID.(activity.Activity), InitiatorID: initiatorID, TargetID: targetID, AccountID: accountID,