extern/netbird
1
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2025-06-19 17:31:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

[management] Migrate events sqlite store to gorm (#3837)

Browse Source
This commit is contained in:
Bethuel Mmbaga 2025-05-20 17:00:37 +03:00 committed by GitHub
parent 1d4cfb83e7
commit 4785f23fc4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 365 additions and 399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -59,7 +59,6 @@ require (
github.com/hashicorp/go-version v1.6.0
github.com/libdns/route53 v1.5.0
github.com/libp2p/go-netroute v0.2.1
github.com/mattn/go-sqlite3 v1.14.22
github.com/mdlayher/socket v0.5.1
github.com/miekg/dns v1.1.59
github.com/mitchellh/hashstructure/v2 v2.0.2
@ -195,6 +194,7 @@ require (
github.com/libdns/libdns v0.2.2 // indirect
github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/mdlayher/genetlink v1.3.2 // indirect
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
github.com/mholt/acmez/v2 v2.0.1 // indirect

17
management/server/activity/event.go
View File

@ -19,22 +19,22 @@ type Event struct {
// Timestamp of the event
Timestamp time.Time
// Activity that was performed during the event
Activity ActivityDescriber
Activity Activity `gorm:"type:integer"`
// ID of the event (can be empty, meaning that it wasn't yet generated)
ID uint64
ID uint64 `gorm:"primaryKey;autoIncrement"`
// InitiatorID is the ID of an object that initiated the event (e.g., a user)
InitiatorID string
// InitiatorName is the name of an object that initiated the event.
InitiatorName string
InitiatorName string `gorm:"-"`
// InitiatorEmail is the email address of an object that initiated the event.
InitiatorEmail string
InitiatorEmail string `gorm:"-"`
// TargetID is the ID of an object that was effected by the event (e.g., a peer)
TargetID string
// AccountID is the ID of an account where the event happened
AccountID string
// Meta of the event, e.g. deleted peer information like name, IP, etc
Meta map[string]any
Meta map[string]any `gorm:"serializer:json"`
}
// Copy the event
@ -57,3 +57,10 @@ func (e *Event) Copy() *Event {
Meta: meta,
}
}
type DeletedUser struct {
ID string `gorm:"primaryKey"`
Email string `gorm:"not null"`
Name string
EncAlgo string `gorm:"not null"`
}

252
management/server/activity/sqlite/migration.go
View File

@ -2,156 +2,180 @@ package sqlite
import (
"context"
"database/sql"
"fmt"
log "github.com/sirupsen/logrus"
"gorm.io/gorm"
"github.com/netbirdio/netbird/management/server/activity"
"github.com/netbirdio/netbird/management/server/migration"
)
func migrate(ctx context.Context, crypt *FieldEncrypt, db *sql.DB) error {
if _, err := db.Exec(createTableQuery); err != nil {
return err
}
func migrate(ctx context.Context, crypt *FieldEncrypt, db *gorm.DB) error {
migrations := getMigrations(ctx, crypt)
if _, err := db.Exec(creatTableDeletedUsersQuery); err != nil {
return err
}
if err := updateDeletedUsersTable(ctx, db); err != nil {
return fmt.Errorf("failed to update deleted_users table: %v", err)
}
return migrateLegacyEncryptedUsersToGCM(ctx, crypt, db)
}
// updateDeletedUsersTable checks and updates the deleted_users table schema to ensure required columns exist.
func updateDeletedUsersTable(ctx context.Context, db *sql.DB) error {
exists, err := checkColumnExists(db, "deleted_users", "name")
if err != nil {
return err
}
if !exists {
log.WithContext(ctx).Debug("Adding name column to the deleted_users table")
_, err = db.Exec(`ALTER TABLE deleted_users ADD COLUMN name TEXT;`)
if err != nil {
for _, m := range migrations {
if err := m(db); err != nil {
return err
}
log.WithContext(ctx).Debug("Successfully added name column to the deleted_users table")
}
exists, err = checkColumnExists(db, "deleted_users", "enc_algo")
if err != nil {
return err
}
if !exists {
log.WithContext(ctx).Debug("Adding enc_algo column to the deleted_users table")
_, err = db.Exec(`ALTER TABLE deleted_users ADD COLUMN enc_algo TEXT;`)
if err != nil {
return err
}
log.WithContext(ctx).Debug("Successfully added enc_algo column to the deleted_users table")
}
return nil
}
// migrateLegacyEncryptedUsersToGCM migrates previously encrypted data using,
type migrationFunc func(*gorm.DB) error
func getMigrations(ctx context.Context, crypt *FieldEncrypt) []migrationFunc {
return []migrationFunc{
func(db *gorm.DB) error {
return migration.MigrateNewField[activity.DeletedUser](ctx, db, "name", "")
},
func(db *gorm.DB) error {
return migration.MigrateNewField[activity.DeletedUser](ctx, db, "enc_algo", "")
},
func(db *gorm.DB) error {
return migrateLegacyEncryptedUsersToGCM(ctx, db, crypt)
},
func(db *gorm.DB) error {
return migrateDuplicateDeletedUsers(ctx, db)
},
}
}
// migrateLegacyEncryptedUsersToGCM migrates previously encrypted data using
// legacy CBC encryption with a static IV to the new GCM encryption method.
func migrateLegacyEncryptedUsersToGCM(ctx context.Context, crypt *FieldEncrypt, db *sql.DB) error {
log.WithContext(ctx).Debug("Migrating CBC encrypted deleted users to GCM")
func migrateLegacyEncryptedUsersToGCM(ctx context.Context, db *gorm.DB, crypt *FieldEncrypt) error {
model := &activity.DeletedUser{}
tx, err := db.Begin()
if err != nil {
return fmt.Errorf("failed to begin transaction: %v", err)
if !db.Migrator().HasTable(model) {
log.WithContext(ctx).Debugf("Table for %T does not exist, no CBC to GCM migration needed", model)
return nil
}
defer func() {
_ = tx.Rollback()
}()
rows, err := tx.Query(fmt.Sprintf(`SELECT id, email, name FROM deleted_users where enc_algo IS NULL OR enc_algo != '%s'`, gcmEncAlgo))
var deletedUsers []activity.DeletedUser
err := db.Model(model).Find(&deletedUsers, "enc_algo IS NULL OR enc_algo != ?", gcmEncAlgo).Error
if err != nil {
return fmt.Errorf("failed to execute select query: %v", err)
return fmt.Errorf("failed to query deleted_users: %w", err)
}
defer rows.Close()
updateStmt, err := tx.Prepare(`UPDATE deleted_users SET email = ?, name = ?, enc_algo = ? WHERE id = ?`)
if err != nil {
return fmt.Errorf("failed to prepare update statement: %v", err)
if len(deletedUsers) == 0 {
log.WithContext(ctx).Debug("No CBC encrypted deleted users to migrate")
return nil
}
defer updateStmt.Close()
if err = processUserRows(ctx, crypt, rows, updateStmt); err != nil {
if err = db.Transaction(func(tx *gorm.DB) error {
for _, user := range deletedUsers {
if err = updateDeletedUserData(tx, user, crypt); err != nil {
return fmt.Errorf("failed to migrate deleted user %s: %w", user.ID, err)
}
}
return nil
}); err != nil {
return err
}
if err = tx.Commit(); err != nil {
return fmt.Errorf("failed to commit transaction: %v", err)
}
log.WithContext(ctx).Debug("Successfully migrated CBC encrypted deleted users to GCM")
return nil
}
// processUserRows processes database rows of user data, decrypts legacy encryption fields, and re-encrypts them using GCM.
func processUserRows(ctx context.Context, crypt *FieldEncrypt, rows *sql.Rows, updateStmt *sql.Stmt) error {
for rows.Next() {
var (
id, decryptedEmail, decryptedName string
email, name *string
)
func updateDeletedUserData(transaction *gorm.DB, user activity.DeletedUser, crypt *FieldEncrypt) error {
var err error
var decryptedEmail, decryptedName string
err := rows.Scan(&id, &email, &name)
if user.Email != "" {
decryptedEmail, err = crypt.LegacyDecrypt(user.Email)
if err != nil {
return err
}
if email != nil {
decryptedEmail, err = crypt.LegacyDecrypt(*email)
if err != nil {
log.WithContext(ctx).Warnf("skipping migrating deleted user %s: %v",
id,
fmt.Errorf("failed to decrypt email: %w", err),
)
continue
}
}
if name != nil {
decryptedName, err = crypt.LegacyDecrypt(*name)
if err != nil {
log.WithContext(ctx).Warnf("skipping migrating deleted user %s: %v",
id,
fmt.Errorf("failed to decrypt name: %w", err),
)
continue
}
}
encryptedEmail, err := crypt.Encrypt(decryptedEmail)
if err != nil {
return fmt.Errorf("failed to encrypt email: %w", err)
}
encryptedName, err := crypt.Encrypt(decryptedName)
if err != nil {
return fmt.Errorf("failed to encrypt name: %w", err)
}
_, err = updateStmt.Exec(encryptedEmail, encryptedName, gcmEncAlgo, id)
if err != nil {
return err
return fmt.Errorf("failed to decrypt email: %w", err)
}
}
if err := rows.Err(); err != nil {
if user.Name != "" {
decryptedName, err = crypt.LegacyDecrypt(user.Name)
if err != nil {
return fmt.Errorf("failed to decrypt name: %w", err)
}
}
updatedUser := user
updatedUser.EncAlgo = gcmEncAlgo
updatedUser.Email, err = crypt.Encrypt(decryptedEmail)
if err != nil {
return fmt.Errorf("failed to encrypt email: %w", err)
}
updatedUser.Name, err = crypt.Encrypt(decryptedName)
if err != nil {
return fmt.Errorf("failed to encrypt name: %w", err)
}
return transaction.Model(&updatedUser).Omit("id").Updates(updatedUser).Error
}
// MigrateDuplicateDeletedUsers removes duplicates and ensures the id column is marked as the primary key
func migrateDuplicateDeletedUsers(ctx context.Context, db *gorm.DB) error {
model := &activity.DeletedUser{}
if !db.Migrator().HasTable(model) {
log.WithContext(ctx).Debugf("Table for %T does not exist, no duplicate migration needed", model)
return nil
}
isPrimaryKey, err := isColumnPrimaryKey[activity.DeletedUser](db, "id")
if err != nil {
return err
}
if isPrimaryKey {
log.WithContext(ctx).Debug("No duplicate deleted users to migrate")
return nil
}
if err = db.Transaction(func(tx *gorm.DB) error {
groupById := tx.Model(model).Select("MAX(rowid)").Group("id")
if err = tx.Delete(model, "rowid NOT IN (?)", groupById).Error; err != nil {
return err
}
if err = tx.Migrator().RenameTable("deleted_users", "deleted_users_old"); err != nil {
return err
}
if err = tx.Migrator().CreateTable(model); err != nil {
return err
}
if err = tx.Exec(`
INSERT INTO deleted_users (id, email, name, enc_algo) SELECT id, email, name, enc_algo
FROM deleted_users_old;`).Error; err != nil {
return err
}
return tx.Migrator().DropTable("deleted_users_old")
}); err != nil {
return err
}
log.WithContext(ctx).Debug("Successfully migrated duplicate deleted users")
return nil
}
// isColumnPrimaryKey checks if a column is a primary key in the given model
func isColumnPrimaryKey[T any](db *gorm.DB, columnName string) (bool, error) {
var model T
cols, err := db.Migrator().ColumnTypes(&model)
if err != nil {
return false, err
}
for _, col := range cols {
if col.Name() == columnName {
isPrimaryKey, _ := col.PrimaryKey()
return isPrimaryKey, nil
}
}
return false, nil
}

148
management/server/activity/sqlite/migration_test.go
View File

@ -2,38 +2,39 @@ package sqlite
import (
"context"
"database/sql"
"path/filepath"
"testing"
"time"
_ "github.com/mattn/go-sqlite3"
"github.com/netbirdio/netbird/management/server/activity"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"github.com/netbirdio/netbird/management/server/activity"
"github.com/netbirdio/netbird/management/server/migration"
)
func setupDatabase(t *testing.T) *sql.DB {
const (
insertDeletedUserQuery = `INSERT INTO deleted_users (id, email, name, enc_algo) VALUES (?, ?, ?, ?)`
)
func setupDatabase(t *testing.T) *gorm.DB {
t.Helper()
dbFile := filepath.Join(t.TempDir(), eventSinkDB)
db, err := sql.Open("sqlite3", dbFile)
require.NoError(t, err, "Failed to open database")
db, err := gorm.Open(sqlite.Open(dbFile))
require.NoError(t, err)
sql, err := db.DB()
require.NoError(t, err)
t.Cleanup(func() {
_ = db.Close()
_ = sql.Close()
})
_, err = db.Exec(createTableQuery)
require.NoError(t, err, "Failed to create events table")
_, err = db.Exec(`CREATE TABLE deleted_users (id TEXT NOT NULL, email TEXT NOT NULL, name TEXT);`)
require.NoError(t, err, "Failed to create deleted_users table")
return db
}
func TestMigrate(t *testing.T) {
func TestMigrateLegacyEncryptedUsersToGCM(t *testing.T) {
db := setupDatabase(t)
key, err := GenerateKey()
@ -42,43 +43,98 @@ func TestMigrate(t *testing.T) {
crypt, err := NewFieldEncrypt(key)
require.NoError(t, err, "Failed to initialize FieldEncrypt")
legacyEmail := crypt.LegacyEncrypt("testaccount@test.com")
legacyName := crypt.LegacyEncrypt("Test Account")
t.Run("empty table, no migration required", func(t *testing.T) {
require.NoError(t, migrateLegacyEncryptedUsersToGCM(context.Background(), db, crypt))
assert.False(t, db.Migrator().HasTable("deleted_users"))
})
_, err = db.Exec(`INSERT INTO events(activity, timestamp, initiator_id, target_id, account_id, meta) VALUES(?, ?, ?, ?, ?, ?)`,
activity.UserDeleted, time.Now(), "initiatorID", "targetID", "accountID", "")
require.NoError(t, err, "Failed to insert event")
require.NoError(t, db.Exec(`CREATE TABLE deleted_users (id TEXT NOT NULL, email TEXT NOT NULL, name TEXT);`).Error)
assert.True(t, db.Migrator().HasTable("deleted_users"))
assert.False(t, db.Migrator().HasColumn("deleted_users", "enc_algo"))
_, err = db.Exec(`INSERT INTO deleted_users(id, email, name) VALUES(?, ?, ?)`, "targetID", legacyEmail, legacyName)
require.NoError(t, err, "Failed to insert legacy encrypted data")
require.NoError(t, migration.MigrateNewField[activity.DeletedUser](context.Background(), db, "enc_algo", ""))
assert.True(t, db.Migrator().HasColumn("deleted_users", "enc_algo"))
colExists, err := checkColumnExists(db, "deleted_users", "enc_algo")
require.NoError(t, err, "Failed to check if enc_algo column exists")
require.False(t, colExists, "enc_algo column should not exist before migration")
t.Run("legacy users migration", func(t *testing.T) {
legacyEmail := crypt.LegacyEncrypt("test.user@test.com")
legacyName := crypt.LegacyEncrypt("Test User")
err = migrate(context.Background(), crypt, db)
require.NoError(t, err, "Migration failed")
require.NoError(t, db.Exec(insertDeletedUserQuery, "user1", legacyEmail, legacyName, "").Error)
require.NoError(t, db.Exec(insertDeletedUserQuery, "user2", legacyEmail, legacyName, "legacy").Error)
colExists, err = checkColumnExists(db, "deleted_users", "enc_algo")
require.NoError(t, err, "Failed to check if enc_algo column exists after migration")
require.True(t, colExists, "enc_algo column should exist after migration")
require.NoError(t, migrateLegacyEncryptedUsersToGCM(context.Background(), db, crypt))
var encAlgo string
err = db.QueryRow(`SELECT enc_algo FROM deleted_users LIMIT 1`, "").Scan(&encAlgo)
require.NoError(t, err, "Failed to select updated data")
require.Equal(t, gcmEncAlgo, encAlgo, "enc_algo should be set to 'GCM' after migration")
var users []activity.DeletedUser
require.NoError(t, db.Find(&users).Error)
assert.Len(t, users, 2)
store, err := createStore(crypt, db)
require.NoError(t, err, "Failed to create store")
for _, user := range users {
assert.Equal(t, gcmEncAlgo, user.EncAlgo)
events, err := store.Get(context.Background(), "accountID", 0, 1, false)
require.NoError(t, err, "Failed to get events")
decryptedEmail, err := crypt.Decrypt(user.Email)
require.NoError(t, err)
assert.Equal(t, "test.user@test.com", decryptedEmail)
require.Len(t, events, 1, "Should have one event")
require.Equal(t, activity.UserDeleted, events[0].Activity, "activity should match")
require.Equal(t, "initiatorID", events[0].InitiatorID, "initiator id should match")
require.Equal(t, "targetID", events[0].TargetID, "target id should match")
require.Equal(t, "accountID", events[0].AccountID, "account id should match")
require.Equal(t, "testaccount@test.com", events[0].Meta["email"], "email should match")
require.Equal(t, "Test Account", events[0].Meta["username"], "username should match")
decryptedName, err := crypt.Decrypt(user.Name)
require.NoError(t, err)
require.Equal(t, "Test User", decryptedName)
}
})
t.Run("users already migrated, no migration", func(t *testing.T) {
encryptedEmail, err := crypt.Encrypt("test.user@test.com")
require.NoError(t, err)
encryptedName, err := crypt.Encrypt("Test User")
require.NoError(t, err)
require.NoError(t, db.Exec(insertDeletedUserQuery, "user3", encryptedEmail, encryptedName, gcmEncAlgo).Error)
require.NoError(t, migrateLegacyEncryptedUsersToGCM(context.Background(), db, crypt))
var users []activity.DeletedUser
require.NoError(t, db.Find(&users).Error)
assert.Len(t, users, 3)
for _, user := range users {
assert.Equal(t, gcmEncAlgo, user.EncAlgo)
decryptedEmail, err := crypt.Decrypt(user.Email)
require.NoError(t, err)
assert.Equal(t, "test.user@test.com", decryptedEmail)
decryptedName, err := crypt.Decrypt(user.Name)
require.NoError(t, err)
require.Equal(t, "Test User", decryptedName)
}
})
}
func TestMigrateDuplicateDeletedUsers(t *testing.T) {
db := setupDatabase(t)
require.NoError(t, migrateDuplicateDeletedUsers(context.Background(), db))
assert.False(t, db.Migrator().HasTable("deleted_users"))
require.NoError(t, db.Exec(`CREATE TABLE deleted_users (id TEXT NOT NULL, email TEXT NOT NULL, name TEXT, enc_algo TEXT NOT NULL);`).Error)
assert.True(t, db.Migrator().HasTable("deleted_users"))
isPrimaryKey, err := isColumnPrimaryKey[activity.DeletedUser](db, "id")
require.NoError(t, err)
assert.False(t, isPrimaryKey)
require.NoError(t, db.Exec(insertDeletedUserQuery, "user1", "email1", "name1", "GCM").Error)
require.NoError(t, db.Exec(insertDeletedUserQuery, "user1", "email2", "name2", "GCM").Error)
require.NoError(t, migrateDuplicateDeletedUsers(context.Background(), db))
isPrimaryKey, err = isColumnPrimaryKey[activity.DeletedUser](db, "id")
require.NoError(t, err)
assert.True(t, isPrimaryKey)
var users []activity.DeletedUser
require.NoError(t, db.Find(&users).Error)
assert.Len(t, users, 1)
assert.Equal(t, "user1", users[0].ID)
assert.Equal(t, "email2", users[0].Email)
assert.Equal(t, "name2", users[0].Name)
assert.Equal(t, "GCM", users[0].EncAlgo)
}

343
management/server/activity/sqlite/sqlite.go
View File

@ -2,75 +2,21 @@ package sqlite
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"path/filepath"
"runtime"
"time"
_ "github.com/mattn/go-sqlite3"
log "github.com/sirupsen/logrus"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"gorm.io/gorm/logger"
"github.com/netbirdio/netbird/management/server/activity"
)
const (
// eventSinkDB is the default name of the events database
eventSinkDB = "events.db"
createTableQuery = "CREATE TABLE IF NOT EXISTS events " +
"(id INTEGER PRIMARY KEY AUTOINCREMENT, " +
"activity INTEGER, " +
"timestamp DATETIME, " +
"initiator_id TEXT," +
"account_id TEXT," +
"meta TEXT," +
" target_id TEXT);"
creatTableDeletedUsersQuery = `CREATE TABLE IF NOT EXISTS deleted_users (id TEXT NOT NULL, email TEXT NOT NULL, name TEXT, enc_algo TEXT NOT NULL);`
selectDescQuery = `SELECT events.id, activity, timestamp, initiator_id, i.name as "initiator_name", i.email as "initiator_email", target_id, t.name as "target_name", t.email as "target_email", account_id, meta
FROM events
LEFT JOIN (
SELECT id, MAX(name) as name, MAX(email) as email
FROM deleted_users
GROUP BY id
) i ON events.initiator_id = i.id
LEFT JOIN (
SELECT id, MAX(name) as name, MAX(email) as email
FROM deleted_users
GROUP BY id
) t ON events.target_id = t.id
WHERE account_id = ?
ORDER BY timestamp DESC LIMIT ? OFFSET ?;`
selectAscQuery = `SELECT events.id, activity, timestamp, initiator_id, i.name as "initiator_name", i.email as "initiator_email", target_id, t.name as "target_name", t.email as "target_email", account_id, meta
FROM events
LEFT JOIN (
SELECT id, MAX(name) as name, MAX(email) as email
FROM deleted_users
GROUP BY id
) i ON events.initiator_id = i.id
LEFT JOIN (
SELECT id, MAX(name) as name, MAX(email) as email
FROM deleted_users
GROUP BY id
) t ON events.target_id = t.id
WHERE account_id = ?
ORDER BY timestamp ASC LIMIT ? OFFSET ?;`
insertQuery = "INSERT INTO events(activity, timestamp, initiator_id, target_id, account_id, meta) " +
"VALUES(?, ?, ?, ?, ?, ?)"
/*
TODO:
The insert should avoid duplicated IDs in the table. So the query should be changes to something like:
`INSERT INTO deleted_users(id, email, name) VALUES(?, ?, ?) ON CONFLICT (id) DO UPDATE SET email = EXCLUDED.email, name = EXCLUDED.name;`
For this to work we have to set the id column as primary key. But this is not possible because the id column is not unique
and some selfhosted deployments might have duplicates already so we need to clean the table first.
*/
insertDeleteUserQuery = `INSERT INTO deleted_users(id, email, name, enc_algo) VALUES(?, ?, ?, ?)`
eventSinkDB = "events.db"
fallbackName = "unknown"
fallbackEmail = "unknown@unknown.com"
@ -78,172 +24,158 @@ const (
gcmEncAlgo = "GCM"
)
type eventWithNames struct {
activity.Event
InitiatorName string
InitiatorEmail string
TargetName string
TargetEmail string
}
// Store is the implementation of the activity.Store interface backed by SQLite
type Store struct {
db *sql.DB
db *gorm.DB
fieldEncrypt *FieldEncrypt
insertStatement *sql.Stmt
selectAscStatement *sql.Stmt
selectDescStatement *sql.Stmt
deleteUserStmt *sql.Stmt
}
// NewSQLiteStore creates a new Store with an event table if not exists.
func NewSQLiteStore(ctx context.Context, dataDir string, encryptionKey string) (*Store, error) {
dbFile := filepath.Join(dataDir, eventSinkDB)
db, err := sql.Open("sqlite3", dbFile)
if err != nil {
return nil, err
}
db.SetMaxOpenConns(runtime.NumCPU())
crypt, err := NewFieldEncrypt(encryptionKey)
if err != nil {
_ = db.Close()
return nil, err
}
dbFile := filepath.Join(dataDir, eventSinkDB)
db, err := gorm.Open(sqlite.Open(dbFile), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
})
if err != nil {
return nil, err
}
sql, err := db.DB()
if err != nil {
return nil, err
}
sql.SetMaxOpenConns(1)
if err = migrate(ctx, crypt, db); err != nil {
_ = db.Close()
return nil, fmt.Errorf("events database migration: %w", err)
}
return createStore(crypt, db)
err = db.AutoMigrate(&activity.Event{}, &activity.DeletedUser{})
if err != nil {
return nil, fmt.Errorf("events auto migrate: %w", err)
}
return &Store{
db: db,
fieldEncrypt: crypt,
}, nil
}
func (store *Store) processResult(ctx context.Context, result *sql.Rows) ([]*activity.Event, error) {
events := make([]*activity.Event, 0)
func (store *Store) processResult(ctx context.Context, events []*eventWithNames) ([]*activity.Event, error) {
activityEvents := make([]*activity.Event, 0)
var cryptErr error
for result.Next() {
var id int64
var operation activity.Activity
var timestamp time.Time
var initiator string
var initiatorName *string
var initiatorEmail *string
var target string
var targetUserName *string
var targetEmail *string
var account string
var jsonMeta string
err := result.Scan(&id, &operation, &timestamp, &initiator, &initiatorName, &initiatorEmail, &target, &targetUserName, &targetEmail, &account, &jsonMeta)
if err != nil {
return nil, err
for _, event := range events {
e := event.Event
if e.Meta == nil {
e.Meta = make(map[string]any)
}
meta := make(map[string]any)
if jsonMeta != "" {
err = json.Unmarshal([]byte(jsonMeta), &meta)
if event.TargetName != "" {
name, err := store.fieldEncrypt.Decrypt(event.TargetName)
if err != nil {
return nil, err
}
}
if targetUserName != nil {
name, err := store.fieldEncrypt.Decrypt(*targetUserName)
if err != nil {
cryptErr = fmt.Errorf("failed to decrypt username for target id: %s", target)
meta["username"] = fallbackName
cryptErr = fmt.Errorf("failed to decrypt username for target id: %s", event.TargetName)
e.Meta["username"] = fallbackName
} else {
meta["username"] = name
e.Meta["username"] = name
}
}
if targetEmail != nil {
email, err := store.fieldEncrypt.Decrypt(*targetEmail)
if event.TargetEmail != "" {
email, err := store.fieldEncrypt.Decrypt(event.TargetEmail)
if err != nil {
cryptErr = fmt.Errorf("failed to decrypt email address for target id: %s", target)
meta["email"] = fallbackEmail
cryptErr = fmt.Errorf("failed to decrypt email address for target id: %s", event.TargetEmail)
e.Meta["email"] = fallbackEmail
} else {
meta["email"] = email
e.Meta["email"] = email
}
}
event := &activity.Event{
Timestamp: timestamp,
Activity: operation,
ID: uint64(id),
InitiatorID: initiator,
TargetID: target,
AccountID: account,
Meta: meta,
}
if initiatorName != nil {
name, err := store.fieldEncrypt.Decrypt(*initiatorName)
if event.InitiatorName != "" {
name, err := store.fieldEncrypt.Decrypt(event.InitiatorName)
if err != nil {
cryptErr = fmt.Errorf("failed to decrypt username of initiator: %s", initiator)
event.InitiatorName = fallbackName
cryptErr = fmt.Errorf("failed to decrypt username of initiator: %s", event.InitiatorName)
e.InitiatorName = fallbackName
} else {
event.InitiatorName = name
e.InitiatorName = name
}
}
if initiatorEmail != nil {
email, err := store.fieldEncrypt.Decrypt(*initiatorEmail)
if event.InitiatorEmail != "" {
email, err := store.fieldEncrypt.Decrypt(event.InitiatorEmail)
if err != nil {
cryptErr = fmt.Errorf("failed to decrypt email address of initiator: %s", initiator)
event.InitiatorEmail = fallbackEmail
cryptErr = fmt.Errorf("failed to decrypt email address of initiator: %s", event.InitiatorEmail)
e.InitiatorEmail = fallbackEmail
} else {
event.InitiatorEmail = email
e.InitiatorEmail = email
}
}
events = append(events, event)
activityEvents = append(activityEvents, &e)
}
if cryptErr != nil {
log.WithContext(ctx).Warnf("%s", cryptErr)
}
return events, nil
return activityEvents, nil
}
// Get returns "limit" number of events from index ordered descending or ascending by a timestamp
func (store *Store) Get(ctx context.Context, accountID string, offset, limit int, descending bool) ([]*activity.Event, error) {
stmt := store.selectDescStatement
baseQuery := store.db.Model(&activity.Event{}).
Select(`
events.*,
u.name AS initiator_name,
u.email AS initiator_email,
t.name AS target_name,
t.email AS target_email
`).
Joins(`LEFT JOIN deleted_users u ON u.id = events.initiator_id`).
Joins(`LEFT JOIN deleted_users t ON t.id = events.target_id`)
orderDir := "DESC"
if !descending {
stmt = store.selectAscStatement
orderDir = "ASC"
}
result, err := stmt.Query(accountID, limit, offset)
var events []*eventWithNames
err := baseQuery.Order("events.timestamp "+orderDir).Offset(offset).Limit(limit).
Find(&events, "account_id = ?", accountID).Error
if err != nil {
return nil, err
}
defer result.Close() //nolint
return store.processResult(ctx, result)
return store.processResult(ctx, events)
}
// Save an event in the SQLite events table end encrypt the "email" element in meta map
func (store *Store) Save(_ context.Context, event *activity.Event) (*activity.Event, error) {
var jsonMeta string
meta, err := store.saveDeletedUserEmailAndNameInEncrypted(event)
if err != nil {
return nil, err
}
if meta != nil {
metaBytes, err := json.Marshal(event.Meta)
if err != nil {
return nil, err
}
jsonMeta = string(metaBytes)
}
result, err := store.insertStatement.Exec(event.Activity, event.Timestamp, event.InitiatorID, event.TargetID, event.AccountID, jsonMeta)
if err != nil {
return nil, err
}
id, err := result.LastInsertId()
if err != nil {
return nil, err
}
eventCopy := event.Copy()
eventCopy.ID = uint64(id)
meta, err := store.saveDeletedUserEmailAndNameInEncrypted(eventCopy)
if err != nil {
return nil, err
}
eventCopy.Meta = meta
if err = store.db.Create(eventCopy).Error; err != nil {
return nil, err
}
return eventCopy, nil
}
@ -260,16 +192,27 @@ func (store *Store) saveDeletedUserEmailAndNameInEncrypted(event *activity.Event
return event.Meta, nil
}
deletedUser := activity.DeletedUser{
ID: event.TargetID,
EncAlgo: gcmEncAlgo,
}
encryptedEmail, err := store.fieldEncrypt.Encrypt(fmt.Sprintf("%s", email))
if err != nil {
return nil, err
}
deletedUser.Email = encryptedEmail
encryptedName, err := store.fieldEncrypt.Encrypt(fmt.Sprintf("%s", name))
if err != nil {
return nil, err
}
deletedUser.Name = encryptedName
_, err = store.deleteUserStmt.Exec(event.TargetID, encryptedEmail, encryptedName, gcmEncAlgo)
err = store.db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "id"}},
DoUpdates: clause.AssignmentColumns([]string{"email", "name"}),
}).Create(deletedUser).Error
if err != nil {
return nil, err
}
@ -285,75 +228,11 @@ func (store *Store) saveDeletedUserEmailAndNameInEncrypted(event *activity.Event
// Close the Store
func (store *Store) Close(_ context.Context) error {
if store.db != nil {
return store.db.Close()
sql, err := store.db.DB()
if err != nil {
return err
}
return sql.Close()
}
return nil
}
// createStore initializes and returns a new Store instance with prepared SQL statements.
func createStore(crypt *FieldEncrypt, db *sql.DB) (*Store, error) {
insertStmt, err := db.Prepare(insertQuery)
if err != nil {
_ = db.Close()
return nil, err
}
selectDescStmt, err := db.Prepare(selectDescQuery)
if err != nil {
_ = db.Close()
return nil, err
}
selectAscStmt, err := db.Prepare(selectAscQuery)
if err != nil {
_ = db.Close()
return nil, err
}
deleteUserStmt, err := db.Prepare(insertDeleteUserQuery)
if err != nil {
_ = db.Close()
return nil, err
}
return &Store{
db: db,
fieldEncrypt: crypt,
insertStatement: insertStmt,
selectDescStatement: selectDescStmt,
selectAscStatement: selectAscStmt,
deleteUserStmt: deleteUserStmt,
}, nil
}
// checkColumnExists checks if a column exists in a specified table
func checkColumnExists(db *sql.DB, tableName, columnName string) (bool, error) {
query := fmt.Sprintf("PRAGMA table_info(%s);", tableName)
rows, err := db.Query(query)
if err != nil {
return false, fmt.Errorf("failed to query table info: %w", err)
}
defer rows.Close()
for rows.Next() {
var cid int
var name, ctype string
var notnull, pk int
var dfltValue sql.NullString
err = rows.Scan(&cid, &name, &ctype, &notnull, &dfltValue, &pk)
if err != nil {
return false, fmt.Errorf("failed to scan row: %w", err)
}
if name == columnName {
return true, nil
}
}
if err = rows.Err(); err != nil {
return false, err
}
return false, nil
}

2
management/server/event.go
View File

@ -66,7 +66,7 @@ func (am *DefaultAccountManager) StoreEvent(ctx context.Context, initiatorID, ta
go func() {
_, err := am.eventStore.Save(ctx, &activity.Event{
Timestamp: time.Now().UTC(),
Activity: activityID,
Activity: activityID.(activity.Activity),
InitiatorID: initiatorID,
TargetID: targetID,
AccountID: accountID,