From 3a01e47701fceb691d162c1a41808455135007c1 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sun, 10 Dec 2023 10:31:38 -0800 Subject: [PATCH] Refactor: Move UsageData and Device to the backend/server directory since they are purely server-side constructs --- backend/server/internal/database/db.go | 6 ++-- backend/server/internal/database/device.go | 28 ++++++++++++++----- .../internal/database/historyentries.go | 2 +- backend/server/internal/database/usagedata.go | 27 ++++++++++++------ .../server/internal/server/api_handlers.go | 3 +- backend/server/internal/server/srv.go | 4 +-- backend/server/server.go | 2 +- shared/data.go | 16 ----------- shared/usagedata.go | 14 ---------- 9 files changed, 48 insertions(+), 54 deletions(-) delete mode 100644 shared/usagedata.go diff --git a/backend/server/internal/database/db.go b/backend/server/internal/database/db.go index cf51bd7..0735fdc 100644 --- a/backend/server/internal/database/db.go +++ b/backend/server/internal/database/db.go @@ -47,8 +47,8 @@ func OpenPostgres(dsn string, config *gorm.Config) (*DB, error) { func (db *DB) AddDatabaseTables() error { models := []any{ &shared.EncHistoryEntry{}, - &shared.Device{}, - &shared.UsageData{}, + &Device{}, + &UsageData{}, &shared.DumpRequest{}, &shared.DeletionRequest{}, &shared.Feedback{}, @@ -216,7 +216,7 @@ func (db *DB) UninstallDevice(ctx context.Context, userId, deviceId string) (int return 0, fmt.Errorf("UninstallDevice: failed to delete dump requests: %w", r3.Error) } // Lastly, update the flag so that we know this device has been deleted - r := db.WithContext(ctx).Model(&shared.Device{}).Where("user_id = ? AND device_id = ?", userId, deviceId).Update("uninstall_date", time.Now().UTC()) + r := db.WithContext(ctx).Model(&Device{}).Where("user_id = ? AND device_id = ?", userId, deviceId).Update("uninstall_date", time.Now().UTC()) if r.Error != nil { return 0, fmt.Errorf("UnisntallDevice: failed to update uninstall_date: %w", r.Error) } diff --git a/backend/server/internal/database/device.go b/backend/server/internal/database/device.go index f8da3d7..f817deb 100644 --- a/backend/server/internal/database/device.go +++ b/backend/server/internal/database/device.go @@ -3,13 +3,27 @@ package database import ( "context" "fmt" - - "github.com/ddworken/hishtory/shared" + "time" ) +type Device struct { + UserId string `json:"user_id"` + DeviceId string `json:"device_id"` + // The IP address that was used to register the device. Recorded so + // that I can count how many people are using hishtory and roughly + // from where. If you would like this deleted, please email me at + // david@daviddworken.com and I can clear it from your device entries. + RegistrationIp string `json:"registration_ip"` + RegistrationDate time.Time `json:"registration_date"` + // Test devices, that should be aggressively cleaned from the DB + IsIntegrationTestDevice bool `json:"is_integration_test_device"` + // Whether this device was uninstalled + UninstallDate time.Time `json:"uninstall_date"` +} + func (db *DB) CountAllDevices(ctx context.Context) (int64, error) { var numDevices int64 = 0 - tx := db.WithContext(ctx).Model(&shared.Device{}).Count(&numDevices) + tx := db.WithContext(ctx).Model(&Device{}).Count(&numDevices) if tx.Error != nil { return 0, fmt.Errorf("tx.Error: %w", tx.Error) } @@ -19,7 +33,7 @@ func (db *DB) CountAllDevices(ctx context.Context) (int64, error) { func (db *DB) CountDevicesForUser(ctx context.Context, userID string) (int64, error) { var existingDevicesCount int64 - tx := db.WithContext(ctx).Model(&shared.Device{}).Where("user_id = ?", userID).Count(&existingDevicesCount) + tx := db.WithContext(ctx).Model(&Device{}).Where("user_id = ?", userID).Count(&existingDevicesCount) if tx.Error != nil { return 0, fmt.Errorf("tx.Error: %w", tx.Error) } @@ -27,7 +41,7 @@ func (db *DB) CountDevicesForUser(ctx context.Context, userID string) (int64, er return existingDevicesCount, nil } -func (db *DB) CreateDevice(ctx context.Context, device *shared.Device) error { +func (db *DB) CreateDevice(ctx context.Context, device *Device) error { tx := db.WithContext(ctx).Create(device) if tx.Error != nil { return fmt.Errorf("tx.Error: %w", tx.Error) @@ -36,8 +50,8 @@ func (db *DB) CreateDevice(ctx context.Context, device *shared.Device) error { return nil } -func (db *DB) DevicesForUser(ctx context.Context, userID string) ([]*shared.Device, error) { - var devices []*shared.Device +func (db *DB) DevicesForUser(ctx context.Context, userID string) ([]*Device, error) { + var devices []*Device tx := db.WithContext(ctx).Where("user_id = ?", userID).Find(&devices) if tx.Error != nil { return nil, fmt.Errorf("tx.Error: %w", tx.Error) diff --git a/backend/server/internal/database/historyentries.go b/backend/server/internal/database/historyentries.go index a303a42..98ef50b 100644 --- a/backend/server/internal/database/historyentries.go +++ b/backend/server/internal/database/historyentries.go @@ -52,7 +52,7 @@ func (db *DB) AddHistoryEntries(ctx context.Context, entries ...*shared.EncHisto }) } -func (db *DB) AddHistoryEntriesForAllDevices(ctx context.Context, devices []*shared.Device, entries []*shared.EncHistoryEntry) error { +func (db *DB) AddHistoryEntriesForAllDevices(ctx context.Context, devices []*Device, entries []*shared.EncHistoryEntry) error { chunkSize := 1000 return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { for _, device := range devices { diff --git a/backend/server/internal/database/usagedata.go b/backend/server/internal/database/usagedata.go index ae86f9e..0a27067 100644 --- a/backend/server/internal/database/usagedata.go +++ b/backend/server/internal/database/usagedata.go @@ -4,12 +4,21 @@ import ( "context" "fmt" "time" - - "github.com/ddworken/hishtory/shared" ) -func (db *DB) UsageDataFindByUserAndDevice(ctx context.Context, userId, deviceId string) ([]shared.UsageData, error) { - var usageData []shared.UsageData +type UsageData struct { + UserId string `json:"user_id" gorm:"not null; uniqueIndex:usageDataUniqueIndex"` + DeviceId string `json:"device_id" gorm:"not null; uniqueIndex:usageDataUniqueIndex"` + LastUsed time.Time `json:"last_used"` + LastIp string `json:"last_ip"` + NumEntriesHandled int `json:"num_entries_handled"` + LastQueried time.Time `json:"last_queried"` + NumQueries int `json:"num_queries"` + Version string `json:"version"` +} + +func (db *DB) UsageDataFindByUserAndDevice(ctx context.Context, userId, deviceId string) ([]UsageData, error) { + var usageData []UsageData tx := db.DB.WithContext(ctx).Where("user_id = ? AND device_id = ?", userId, deviceId).Find(&usageData) if tx.Error != nil { @@ -23,7 +32,7 @@ func (db *DB) UsageDataFindByUserAndDevice(ctx context.Context, userId, deviceId return usageData, nil } -func (db *DB) CreateUsageData(ctx context.Context, usageData *shared.UsageData) error { +func (db *DB) CreateUsageData(ctx context.Context, usageData *UsageData) error { tx := db.DB.WithContext(ctx).Create(usageData) if tx.Error != nil { return fmt.Errorf("db.WithContext.Create: %w", tx.Error) @@ -34,7 +43,7 @@ func (db *DB) CreateUsageData(ctx context.Context, usageData *shared.UsageData) // UpdateUsageData updates the entry for a given userID/deviceID pair with the lastUsed and lastIP values func (db *DB) UpdateUsageData(ctx context.Context, userId, deviceId string, lastUsed time.Time, lastIP string) error { - tx := db.DB.WithContext(ctx).Model(&shared.UsageData{}). + tx := db.DB.WithContext(ctx).Model(&UsageData{}). Where("user_id = ? AND device_id = ?", userId, deviceId). Update("last_used", lastUsed). Update("last_ip", lastIP) @@ -141,7 +150,7 @@ func (db *DB) UsageDataTotal(ctx context.Context) (int64, error) { } nep := numEntriesProcessed{} - tx := db.WithContext(ctx).Model(&shared.UsageData{}).Select("SUM(num_entries_handled) as total").Find(&nep) + tx := db.WithContext(ctx).Model(&UsageData{}).Select("SUM(num_entries_handled) as total").Find(&nep) if tx.Error != nil { return 0, fmt.Errorf("tx.Error: %w", tx.Error) } @@ -151,7 +160,7 @@ func (db *DB) UsageDataTotal(ctx context.Context) (int64, error) { func (db *DB) CountActiveInstalls(ctx context.Context, since time.Duration) (int64, error) { var activeInstalls int64 - tx := db.WithContext(ctx).Model(&shared.UsageData{}).Where("last_used > ?", time.Now().Add(-since)).Count(&activeInstalls) + tx := db.WithContext(ctx).Model(&UsageData{}).Where("last_used > ?", time.Now().Add(-since)).Count(&activeInstalls) if tx.Error != nil { return 0, fmt.Errorf("tx.Error: %w", tx.Error) } @@ -161,7 +170,7 @@ func (db *DB) CountActiveInstalls(ctx context.Context, since time.Duration) (int func (db *DB) CountQueryUsers(ctx context.Context, since time.Duration) (int64, error) { var activeQueryUsers int64 - tx := db.WithContext(ctx).Model(&shared.UsageData{}).Where("last_queried > ?", time.Now().Add(-since)).Count(&activeQueryUsers) + tx := db.WithContext(ctx).Model(&UsageData{}).Where("last_queried > ?", time.Now().Add(-since)).Count(&activeQueryUsers) if tx.Error != nil { return 0, fmt.Errorf("tx.Error: %w", tx.Error) } diff --git a/backend/server/internal/server/api_handlers.go b/backend/server/internal/server/api_handlers.go index dc79912..5ce435f 100644 --- a/backend/server/internal/server/api_handlers.go +++ b/backend/server/internal/server/api_handlers.go @@ -9,6 +9,7 @@ import ( "net/http" "time" + "github.com/ddworken/hishtory/backend/server/internal/database" "github.com/ddworken/hishtory/shared" "github.com/ddworken/hishtory/shared/ai" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" @@ -216,7 +217,7 @@ func (s *Server) apiRegisterHandler(w http.ResponseWriter, r *http.Request) { existingDevicesCount, err := s.db.CountDevicesForUser(r.Context(), userId) checkGormError(err) fmt.Printf("apiRegisterHandler: existingDevicesCount=%d\n", existingDevicesCount) - if err := s.db.CreateDevice(r.Context(), &shared.Device{UserId: userId, DeviceId: deviceId, RegistrationIp: getRemoteAddr(r), RegistrationDate: time.Now(), IsIntegrationTestDevice: isIntegrationTestDevice}); err != nil { + if err := s.db.CreateDevice(r.Context(), &database.Device{UserId: userId, DeviceId: deviceId, RegistrationIp: getRemoteAddr(r), RegistrationDate: time.Now(), IsIntegrationTestDevice: isIntegrationTestDevice}); err != nil { checkGormError(err) } diff --git a/backend/server/internal/server/srv.go b/backend/server/internal/server/srv.go index 423457c..5e2bcbc 100644 --- a/backend/server/internal/server/srv.go +++ b/backend/server/internal/server/srv.go @@ -158,7 +158,7 @@ func (s *Server) updateUsageData(ctx context.Context, version string, remoteAddr if !s.trackUsageData { return nil } - var usageData []shared.UsageData + var usageData []database.UsageData usageData, err := s.db.UsageDataFindByUserAndDevice(ctx, userId, deviceId) if err != nil && !strings.Contains(err.Error(), "record not found") { return fmt.Errorf("db.UsageDataFindByUserAndDevice: %w", err) @@ -166,7 +166,7 @@ func (s *Server) updateUsageData(ctx context.Context, version string, remoteAddr if len(usageData) == 0 { err := s.db.CreateUsageData( ctx, - &shared.UsageData{ + &database.UsageData{ UserId: userId, DeviceId: deviceId, LastUsed: time.Now(), diff --git a/backend/server/server.go b/backend/server/server.go index 4fdde5e..fbcb54a 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -90,8 +90,8 @@ func OpenDB() (*database.DB, error) { return nil, fmt.Errorf("failed to connect to the DB: %w", err) } } - fmt.Println("AutoMigrating DB tables") if !isProductionEnvironment() { + fmt.Println("AutoMigrating DB tables") err := db.AddDatabaseTables() if err != nil { return nil, fmt.Errorf("failed to create underlying DB tables: %w", err) diff --git a/shared/data.go b/shared/data.go index 2589e0c..5cd5aef 100644 --- a/shared/data.go +++ b/shared/data.go @@ -21,22 +21,6 @@ type EncHistoryEntry struct { ReadCount int `json:"read_count"` } -// TODO: Move this struct to the backend directory -type Device struct { - UserId string `json:"user_id"` - DeviceId string `json:"device_id"` - // The IP address that was used to register the device. Recorded so - // that I can count how many people are using hishtory and roughly - // from where. If you would like this deleted, please email me at - // david@daviddworken.com and I can clear it from your device entries. - RegistrationIp string `json:"registration_ip"` - RegistrationDate time.Time `json:"registration_date"` - // Test devices, that should be aggressively cleaned from the DB - IsIntegrationTestDevice bool `json:"is_integration_test_device"` - // Whether this device was uninstalled - UninstallDate time.Time `json:"uninstall_date"` -} - // Represents a request to get all history entries from a given device. Used as part of bootstrapping // a new device. type DumpRequest struct { diff --git a/shared/usagedata.go b/shared/usagedata.go deleted file mode 100644 index cc515fb..0000000 --- a/shared/usagedata.go +++ /dev/null @@ -1,14 +0,0 @@ -package shared - -import "time" - -type UsageData struct { - UserId string `json:"user_id" gorm:"not null; uniqueIndex:usageDataUniqueIndex"` - DeviceId string `json:"device_id" gorm:"not null; uniqueIndex:usageDataUniqueIndex"` - LastUsed time.Time `json:"last_used"` - LastIp string `json:"last_ip"` - NumEntriesHandled int `json:"num_entries_handled"` - LastQueried time.Time `json:"last_queried"` - NumQueries int `json:"num_queries"` - Version string `json:"version"` -}