From a436edbd16869bf5a14377cb425202e2dca4efaa Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sat, 16 Apr 2022 11:37:43 -0700 Subject: [PATCH] Add backend code to delete entries that have already been read + start collecitng data on usage data so we can eventually prune data from users that are no longer using hishtory --- backend/server/server.go | 40 +++++++++++++++++++++++++++++++++-- backend/server/server_test.go | 3 +++ hishtory.go | 3 ++- shared/testutils.go | 3 +++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/backend/server/server.go b/backend/server/server.go index 3384278..1f7ec2b 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -28,6 +28,22 @@ var ( ReleaseVersion string = "UNKNOWN" ) +type UsageData struct { + UserId string `json:"user_id" gorm:"not null; uniqueIndex:usageDataUniqueIndex"` + DeviceId string `json:"device_id" gorm:"not null; uniqueIndex:usageDataUniqueIndex"` + LastUsed time.Time `json:"last_used"` +} + +func updateUsageData(userId, deviceId string) { + var usageData []UsageData + GLOBAL_DB.Where("user_id = ? AND device_id = ?", userId, deviceId).Find(&usageData) + if len(usageData) == 0 { + GLOBAL_DB.Create(&UsageData{UserId: userId, DeviceId: deviceId, LastUsed: time.Now()}) + } else { + GLOBAL_DB.Model(&UsageData{}).Where("user_id = ? AND device_id = ?", userId, deviceId).Update("last_used", time.Now()) + } +} + func apiESubmitHandler(w http.ResponseWriter, r *http.Request) { data, err := ioutil.ReadAll(r.Body) if err != nil { @@ -40,6 +56,7 @@ func apiESubmitHandler(w http.ResponseWriter, r *http.Request) { } fmt.Printf("apiESubmitHandler: received request containg %d EncHistoryEntry\n", len(entries)) for _, entry := range entries { + updateUsageData(entry.UserId, entry.DeviceId) tx := GLOBAL_DB.Where("user_id = ?", entry.UserId) var devices []*shared.Device result := tx.Find(&devices) @@ -61,7 +78,9 @@ func apiESubmitHandler(w http.ResponseWriter, r *http.Request) { } func apiEQueryHandler(w http.ResponseWriter, r *http.Request) { + userId := r.URL.Query().Get("user_id") deviceId := r.URL.Query().Get("device_id") + updateUsageData(userId, deviceId) // Increment the count GLOBAL_DB.Exec("UPDATE enc_history_entries SET read_count = read_count + 1 WHERE device_id = ?", deviceId) @@ -83,6 +102,8 @@ func apiEQueryHandler(w http.ResponseWriter, r *http.Request) { // TODO: bootstrap is a janky solution for the initial version of this. Long term, need to support deleting entries from the DB which means replacing bootstrap with a queued message sent to any live instances. func apiEBootstrapHandler(w http.ResponseWriter, r *http.Request) { userId := r.URL.Query().Get("user_id") + deviceId := r.URL.Query().Get("device_id") + updateUsageData(userId, deviceId) tx := GLOBAL_DB.Where("user_id = ?", userId) var historyEntries []*shared.EncHistoryEntry result := tx.Find(&historyEntries) @@ -100,6 +121,7 @@ func apiERegisterHandler(w http.ResponseWriter, r *http.Request) { userId := r.URL.Query().Get("user_id") deviceId := r.URL.Query().Get("device_id") GLOBAL_DB.Create(&shared.Device{UserId: userId, DeviceId: deviceId, RegistrationIp: r.RemoteAddr, RegistrationDate: time.Now()}) + updateUsageData(userId, deviceId) } func apiBannerHandler(w http.ResponseWriter, r *http.Request) { @@ -122,6 +144,7 @@ func OpenDB() (*gorm.DB, error) { } db.AutoMigrate(&shared.EncHistoryEntry{}) db.AutoMigrate(&shared.Device{}) + db.AutoMigrate(&UsageData{}) return db, nil } @@ -131,6 +154,7 @@ func OpenDB() (*gorm.DB, error) { } db.AutoMigrate(&shared.EncHistoryEntry{}) db.AutoMigrate(&shared.Device{}) + db.AutoMigrate(&UsageData{}) return db, nil } @@ -138,16 +162,20 @@ func init() { if ReleaseVersion == "UNKNOWN" && !isTestEnvironment() { panic("server.go was built without a ReleaseVersion!") } - go keepReleaseVersionUpToDate() InitDB() + go runBackgroundJobs() } -func keepReleaseVersionUpToDate() { +func runBackgroundJobs() { for { err := updateReleaseVersion() if err != nil { fmt.Println(err) } + err = cleanDatabase() + if err != nil { + fmt.Println(err) + } time.Sleep(10 * time.Minute) } } @@ -253,6 +281,14 @@ func byteCountToString(b int) string { return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMG"[exp]) } +func cleanDatabase() error { + result := GLOBAL_DB.Exec("DELETE FROM enc_history_entries WHERE read_count > 10") + if result.Error != nil { + return result.Error + } + return nil +} + func main() { fmt.Println("Listening on localhost:8080") http.Handle("/api/v1/esubmit", withLogging(apiESubmitHandler)) diff --git a/backend/server/server_test.go b/backend/server/server_test.go index 8a041f0..c6eeec7 100644 --- a/backend/server/server_test.go +++ b/backend/server/server_test.go @@ -43,6 +43,7 @@ func TestESubmitThenQuery(t *testing.T) { // Query for device id 1 w := httptest.NewRecorder() + // TODO: update this to include the user ID searchReq := httptest.NewRequest(http.MethodGet, "/?device_id="+devId1, nil) apiEQueryHandler(w, searchReq) res := w.Result() @@ -72,6 +73,7 @@ func TestESubmitThenQuery(t *testing.T) { // Same for device id 2 w = httptest.NewRecorder() + // TODO: update this to include the user ID searchReq = httptest.NewRequest(http.MethodGet, "/?device_id="+devId2, nil) apiEQueryHandler(w, searchReq) res = w.Result() @@ -101,6 +103,7 @@ func TestESubmitThenQuery(t *testing.T) { // Bootstrap handler should return 2 entries, one for each device w = httptest.NewRecorder() searchReq = httptest.NewRequest(http.MethodGet, "/?user_id="+data.UserId("key"), nil) + // TODO: update to include device_id apiEBootstrapHandler(w, searchReq) res = w.Result() defer res.Body.Close() diff --git a/hishtory.go b/hishtory.go index 7e2d6f7..e05cf64 100644 --- a/hishtory.go +++ b/hishtory.go @@ -63,7 +63,7 @@ func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error { if err != nil { return err } - respBody, err := lib.ApiGet("/api/v1/equery?device_id=" + config.DeviceId) + respBody, err := lib.ApiGet("/api/v1/equery?device_id=" + config.DeviceId + "&user_id=" + data.UserId(config.UserSecret)) if err != nil { return err } @@ -131,6 +131,7 @@ func saveHistoryEntry() { // Persist it remotely encEntry, err := data.EncryptHistoryEntry(config.UserSecret, *entry) lib.CheckFatalError(err) + encEntry.DeviceId = config.DeviceId jsonValue, err := json.Marshal([]shared.EncHistoryEntry{encEntry}) lib.CheckFatalError(err) _, err = lib.ApiPost("/api/v1/esubmit", "application/json", jsonValue) diff --git a/shared/testutils.go b/shared/testutils.go index e8ebb59..0b9a55e 100644 --- a/shared/testutils.go +++ b/shared/testutils.go @@ -99,6 +99,9 @@ func RunTestServer() func() { if strings.Contains(stderr.String()+stdout.String(), "failed to") { panic(fmt.Sprintf("server failed to do something: stderr=%#v, stdout=%#v", stderr.String(), stdout.String())) } + if strings.Contains(stderr.String()+stdout.String(), "ERROR:") { + panic(fmt.Sprintf("server experienced an error: stderr=%#v, stdout=%#v", stderr.String(), stdout.String())) + } // fmt.Printf("stderr=%#v, stdout=%#v\n", stderr.String(), stdout.String()) } }