Remove the 'e' prefix from api endpoints + implement backend APIs for clean loading of all data from other instances

backend/server/server.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"os/user"
 	"strconv"
 	"strings"
 	"time"
@@ -45,7 +46,13 @@ func updateUsageData(userId, deviceId string) {
 	}
 }
 
-func apiESubmitHandler(w http.ResponseWriter, r *http.Request) {
+type DumpRequest struct {
+	UserId             string    `json:"user_id"`
+	RequestingDeviceId string    `json:"requesting_device_id"`
+	RequestTime        time.Time `json:"request_time"`
+}
+
+func apiSubmitHandler(w http.ResponseWriter, r *http.Request) {
 	data, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		panic(err)
@@ -55,7 +62,7 @@ func apiESubmitHandler(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		panic(fmt.Sprintf("body=%#v, err=%v", data, err))
 	}
-	fmt.Printf("apiESubmitHandler: received request containg %d EncHistoryEntry\n", len(entries))
+	fmt.Printf("apiSubmitHandler: received request containg %d EncHistoryEntry\n", len(entries))
 	for _, entry := range entries {
 		updateUsageData(entry.UserId, entry.DeviceId)
 		tx := GLOBAL_DB.Where("user_id = ?", entry.UserId)
@@ -67,7 +74,7 @@ func apiESubmitHandler(w http.ResponseWriter, r *http.Request) {
 		if len(devices) == 0 {
 			panic(fmt.Errorf("found no devices associated with user_id=%s, can't save history entry", entry.UserId))
 		}
-		fmt.Printf("apiESubmitHandler: Found %d devices\n", len(devices))
+		fmt.Printf("apiSubmitHandler: Found %d devices\n", len(devices))
 		for _, device := range devices {
 			entry.DeviceId = device.DeviceId
 			result := GLOBAL_DB.Create(&entry)
@@ -78,7 +85,7 @@ func apiESubmitHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func apiEQueryHandler(w http.ResponseWriter, r *http.Request) {
+func apiQueryHandler(w http.ResponseWriter, r *http.Request) {
 	userId := r.URL.Query().Get("user_id")
 	deviceId := r.URL.Query().Get("device_id")
 	updateUsageData(userId, deviceId)
@@ -92,7 +99,7 @@ func apiEQueryHandler(w http.ResponseWriter, r *http.Request) {
 	if result.Error != nil {
 		panic(fmt.Errorf("DB query error: %v", result.Error))
 	}
-	fmt.Printf("apiEQueryHandler: Found %d entries\n", len(historyEntries))
+	fmt.Printf("apiQueryHandler: Found %d entries\n", len(historyEntries))
 	resp, err := json.Marshal(historyEntries)
 	if err != nil {
 		panic(err)
@@ -100,31 +107,63 @@ func apiEQueryHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write(resp)
 }
 
-// TODO: bootstrap is a janky solution for the initial version of this. Long term, need to support deleting entries from the DB which means replacing bootstrap with a queued message sent to any live instances.
-func apiEBootstrapHandler(w http.ResponseWriter, r *http.Request) {
-	userId := r.URL.Query().Get("user_id")
-	deviceId := r.URL.Query().Get("device_id")
-	updateUsageData(userId, deviceId)
-	tx := GLOBAL_DB.Where("user_id = ?", userId)
-	var historyEntries []*shared.EncHistoryEntry
-	result := tx.Find(&historyEntries)
-	if result.Error != nil {
-		panic(fmt.Errorf("DB query error: %v", result.Error))
-	}
-	resp, err := json.Marshal(historyEntries)
-	if err != nil {
-		panic(err)
-	}
-	w.Write(resp)
-}
-
-func apiERegisterHandler(w http.ResponseWriter, r *http.Request) {
+func apiRegisterHandler(w http.ResponseWriter, r *http.Request) {
 	userId := r.URL.Query().Get("user_id")
 	deviceId := r.URL.Query().Get("device_id")
 	GLOBAL_DB.Create(&shared.Device{UserId: userId, DeviceId: deviceId, RegistrationIp: r.RemoteAddr, RegistrationDate: time.Now()})
+	GLOBAL_DB.Create(&DumpRequest{UserId: userId, RequestingDeviceId: deviceId, RequestTime: time.Now()})
 	updateUsageData(userId, deviceId)
 }
 
+func apiGetPendingDumpRequestsHandler(w http.ResponseWriter, r *http.Request) {
+	userId := r.URL.Query().Get("user_id")
+	var dumpRequests []*DumpRequest
+	result := GLOBAL_DB.Where("user_id = ?", userId).Find(&dumpRequests)
+	if result.Error != nil {
+		panic(fmt.Errorf("DB query error: %v", result.Error))
+	}
+	respBody, err := json.Marshal(dumpRequests)
+	if err != nil {
+		panic(fmt.Errorf("failed to JSON marshall the dump requests: %v", err))
+	}
+	w.Write(respBody)
+}
+
+func apiSubmitDumpHandler(w http.ResponseWriter, r *http.Request) {
+	userId := r.URL.Query().Get("user_id")
+	requestingDeviceId := r.URL.Query().Get("requesting_device_id")
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		panic(err)
+	}
+	var entries []shared.EncHistoryEntry
+	err = json.Unmarshal(data, &entries)
+	if err != nil {
+		panic(fmt.Sprintf("body=%#v, err=%v", data, err))
+	}
+	fmt.Printf("apiSubmitDumpHandler: received request containg %d EncHistoryEntry\n", len(entries))
+	err = GLOBAL_DB.Transaction(func(tx *gorm.DB) error {
+		for _, entry := range entries {
+			entry.DeviceId = requestingDeviceId
+			if entry.UserId != userId {
+				return fmt.Errorf("batch contains an entry with UserId=%#v, when the query param contained the user_id=%#v", entry.UserId, userId)
+			}
+			result := tx.Create(&entry)
+			if result.Error != nil {
+				return fmt.Errorf("failed to create entry: %v", err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		panic(fmt.Errorf("failed to execute transaction to add dumped DB: %v", err))
+	}
+	result := GLOBAL_DB.Delete(&DumpRequest{}, "user_id = ? AND requesting_device_id = ?", userId, requestingDeviceId)
+	if result.Error != nil {
+		panic(fmt.Errorf("failed to clear the dump request: %v", err))
+	}
+}
+
 func apiBannerHandler(w http.ResponseWriter, r *http.Request) {
 	commitHash := r.URL.Query().Get("commit_hash")
 	deviceId := r.URL.Query().Get("device_id")
@@ -134,7 +173,11 @@ func apiBannerHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func isTestEnvironment() bool {
-	return os.Getenv("HISHTORY_TEST") != ""
+	u, err := user.Current()
+	if err != nil {
+		panic(err)
+	}
+	return os.Getenv("HISHTORY_TEST") != "" || u.Username == "david"
 }
 
 func OpenDB() (*gorm.DB, error) {
@@ -146,6 +189,8 @@ func OpenDB() (*gorm.DB, error) {
 		db.AutoMigrate(&shared.EncHistoryEntry{})
 		db.AutoMigrate(&shared.Device{})
 		db.AutoMigrate(&UsageData{})
+		db.AutoMigrate(&DumpRequest{})
+		db.Exec("PRAGMA journal_mode = WAL")
 		return db, nil
 	}
 
@@ -156,6 +201,7 @@ func OpenDB() (*gorm.DB, error) {
 	db.AutoMigrate(&shared.EncHistoryEntry{})
 	db.AutoMigrate(&shared.Device{})
 	db.AutoMigrate(&UsageData{})
+	db.AutoMigrate(&DumpRequest{})
 	return db, nil
 }
 
@@ -180,6 +226,7 @@ func cron() error {
 }
 
 func runBackgroundJobs() {
+	time.Sleep(5 * time.Second)
 	for {
 		err := cron()
 		if err != nil {
@@ -374,12 +421,13 @@ func cleanDatabase() error {
 
 func main() {
 	fmt.Println("Listening on localhost:8080")
-	http.Handle("/api/v1/esubmit", withLogging(apiESubmitHandler))
-	http.Handle("/api/v1/equery", withLogging(apiEQueryHandler))
-	http.Handle("/api/v1/ebootstrap", withLogging(apiEBootstrapHandler))
-	http.Handle("/api/v1/eregister", withLogging(apiERegisterHandler))
+	http.Handle("/api/v1/submit", withLogging(apiSubmitHandler))
+	http.Handle("/api/v1/get-dump-requests", withLogging(apiGetPendingDumpRequestsHandler))
+	http.Handle("/api/v1/submit-dump", withLogging(apiSubmitDumpHandler))
+	http.Handle("/api/v1/query", withLogging(apiQueryHandler))
+	http.Handle("/api/v1/register", withLogging(apiRegisterHandler))
 	http.Handle("/api/v1/banner", withLogging(apiBannerHandler))
-	http.Handle("/api/v1/trigger-cron", withLogging(triggerCronHandler))
 	http.Handle("/api/v1/download", withLogging(apiDownloadHandler))
+	http.Handle("/api/v1/trigger-cron", withLogging(triggerCronHandler))
 	log.Fatal(http.ListenAndServe(":8080", nil))
 }

backend/server/server_test.go

@@ -26,11 +26,11 @@ func TestESubmitThenQuery(t *testing.T) {
 	otherUser := data.UserId("otherkey")
 	otherDev := uuid.Must(uuid.NewRandom()).String()
 	deviceReq := httptest.NewRequest(http.MethodGet, "/?device_id="+devId1+"&user_id="+userId, nil)
-	apiERegisterHandler(nil, deviceReq)
+	apiRegisterHandler(nil, deviceReq)
 	deviceReq = httptest.NewRequest(http.MethodGet, "/?device_id="+devId2+"&user_id="+userId, nil)
-	apiERegisterHandler(nil, deviceReq)
+	apiRegisterHandler(nil, deviceReq)
 	deviceReq = httptest.NewRequest(http.MethodGet, "/?device_id="+otherDev+"&user_id="+otherUser, nil)
-	apiERegisterHandler(nil, deviceReq)
+	apiRegisterHandler(nil, deviceReq)
 
 	// Submit a few entries for different devices
 	entry := data.MakeFakeHistoryEntry("ls ~/")
@@ -39,12 +39,12 @@ func TestESubmitThenQuery(t *testing.T) {
 	reqBody, err := json.Marshal([]shared.EncHistoryEntry{encEntry})
 	shared.Check(t, err)
 	submitReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(reqBody))
-	apiESubmitHandler(nil, submitReq)
+	apiSubmitHandler(nil, submitReq)
 
 	// Query for device id 1
 	w := httptest.NewRecorder()
 	searchReq := httptest.NewRequest(http.MethodGet, "/?device_id="+devId1+"&user_id="+userId, nil)
-	apiEQueryHandler(w, searchReq)
+	apiQueryHandler(w, searchReq)
 	res := w.Result()
 	defer res.Body.Close()
 	respBody, err := ioutil.ReadAll(res.Body)
@@ -73,7 +73,7 @@ func TestESubmitThenQuery(t *testing.T) {
 	// Same for device id 2
 	w = httptest.NewRecorder()
 	searchReq = httptest.NewRequest(http.MethodGet, "/?device_id="+devId2+"&user_id="+userId, nil)
-	apiEQueryHandler(w, searchReq)
+	apiQueryHandler(w, searchReq)
 	res = w.Result()
 	defer res.Body.Close()
 	respBody, err = ioutil.ReadAll(res.Body)
@@ -97,19 +97,157 @@ func TestESubmitThenQuery(t *testing.T) {
 	if !data.EntryEquals(decEntry, entry) {
 		t.Fatalf("DB data is different than input! \ndb   =%#v\ninput=%#v", *dbEntry, entry)
 	}
+}
 
-	// Bootstrap handler should return 2 entries, one for each device
+func TestDumpRequestAndResponse(t *testing.T) {
+	// Set up
+	defer shared.BackupAndRestore(t)()
+	InitDB()
+
+	// Register a first device for two different users
+	userId := data.UserId("dkey")
+	devId1 := uuid.Must(uuid.NewRandom()).String()
+	otherUser := data.UserId("dOtherkey")
+	otherDev1 := uuid.Must(uuid.NewRandom()).String()
+	deviceReq := httptest.NewRequest(http.MethodGet, "/?device_id="+devId1+"&user_id="+userId, nil)
+	apiRegisterHandler(nil, deviceReq)
+	deviceReq = httptest.NewRequest(http.MethodGet, "/?device_id="+otherDev1+"&user_id="+otherUser, nil)
+	apiRegisterHandler(nil, deviceReq)
+
+	// Query for dump requests, there should be one for userId
+	w := httptest.NewRecorder()
+	apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/?user_id="+userId, nil))
+	res := w.Result()
+	defer res.Body.Close()
+	respBody, err := ioutil.ReadAll(res.Body)
+	shared.Check(t, err)
+	var dumpRequests []*DumpRequest
+	shared.Check(t, json.Unmarshal(respBody, &dumpRequests))
+	if len(dumpRequests) != 1 {
+		t.Fatalf("expected one pending dump request, got %#v", dumpRequests)
+	}
+	dumpRequest := dumpRequests[0]
+	if dumpRequest.RequestingDeviceId != devId1 {
+		t.Fatalf("unexpected device ID")
+	}
+	if dumpRequest.UserId != userId {
+		t.Fatalf("unexpected user ID")
+	}
+
+	// And one for otherUser
 	w = httptest.NewRecorder()
-	searchReq = httptest.NewRequest(http.MethodGet, "/?user_id="+data.UserId("key")+"&device_id="+devId1, nil)
-	apiEBootstrapHandler(w, searchReq)
+	apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/?user_id="+otherUser, nil))
 	res = w.Result()
 	defer res.Body.Close()
 	respBody, err = ioutil.ReadAll(res.Body)
 	shared.Check(t, err)
+	dumpRequests = make([]*DumpRequest, 0)
+	shared.Check(t, json.Unmarshal(respBody, &dumpRequests))
+	if len(dumpRequests) != 1 {
+		t.Fatalf("expected one pending dump request, got %#v", dumpRequests)
+	}
+	dumpRequest = dumpRequests[0]
+	if dumpRequest.RequestingDeviceId != otherDev1 {
+		t.Fatalf("unexpected device ID")
+	}
+	if dumpRequest.UserId != otherUser {
+		t.Fatalf("unexpected user ID")
+	}
+
+	// And none if we query without a user ID
+	w = httptest.NewRecorder()
+	apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/", nil))
+	res = w.Result()
+	defer res.Body.Close()
+	respBody, err = ioutil.ReadAll(res.Body)
+	shared.Check(t, err)
+	if string(respBody) != "[]" {
+		t.Fatalf("got unexpected respBody: %#v", string(respBody))
+	}
+
+	// And none if we query for a user ID that doesn't exit
+	w = httptest.NewRecorder()
+	apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/?user_id=foo", nil))
+	res = w.Result()
+	defer res.Body.Close()
+	respBody, err = ioutil.ReadAll(res.Body)
+	shared.Check(t, err)
+	if string(respBody) != "[]" {
+		t.Fatalf("got unexpected respBody: %#v", string(respBody))
+	}
+
+	// Now submit a dump for userId
+	entry1Dec := data.MakeFakeHistoryEntry("ls ~/")
+	entry1, err := data.EncryptHistoryEntry("dkey", entry1Dec)
+	shared.Check(t, err)
+	entry2Dec := data.MakeFakeHistoryEntry("aaaaaaáaaa")
+	entry2, err := data.EncryptHistoryEntry("dkey", entry1Dec)
+	shared.Check(t, err)
+	reqBody, err := json.Marshal([]shared.EncHistoryEntry{entry1, entry2})
+	shared.Check(t, err)
+	submitReq := httptest.NewRequest(http.MethodPost, "/?user_id="+userId+"&requesting_device_id="+devId1, bytes.NewReader(reqBody))
+	apiSubmitDumpHandler(nil, submitReq)
+
+	// Check that the dump request is no longer there for userId
+	w = httptest.NewRecorder()
+	apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/?user_id="+userId, nil))
+	res = w.Result()
+	defer res.Body.Close()
+	respBody, err = ioutil.ReadAll(res.Body)
+	shared.Check(t, err)
+	if string(respBody) != "[]" {
+		t.Fatalf("got unexpected respBody: %#v", string(respBody))
+	}
+
+	// But it is there for the other user
+	w = httptest.NewRecorder()
+	apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/?user_id="+otherUser, nil))
+	res = w.Result()
+	defer res.Body.Close()
+	respBody, err = ioutil.ReadAll(res.Body)
+	shared.Check(t, err)
+	dumpRequests = make([]*DumpRequest, 0)
+	shared.Check(t, json.Unmarshal(respBody, &dumpRequests))
+	if len(dumpRequests) != 1 {
+		t.Fatalf("expected one pending dump request, got %#v", dumpRequests)
+	}
+	dumpRequest = dumpRequests[0]
+	if dumpRequest.RequestingDeviceId != otherDev1 {
+		t.Fatalf("unexpected device ID")
+	}
+	if dumpRequest.UserId != otherUser {
+		t.Fatalf("unexpected user ID")
+	}
+
+	// And finally, query to ensure that the dumped entries are in the DB
+	w = httptest.NewRecorder()
+	searchReq := httptest.NewRequest(http.MethodGet, "/?device_id="+devId1+"&user_id="+userId, nil)
+	apiQueryHandler(w, searchReq)
+	res = w.Result()
+	defer res.Body.Close()
+	respBody, err = ioutil.ReadAll(res.Body)
+	shared.Check(t, err)
+	var retrievedEntries []*shared.EncHistoryEntry
 	shared.Check(t, json.Unmarshal(respBody, &retrievedEntries))
 	if len(retrievedEntries) != 2 {
 		t.Fatalf("Expected to retrieve 2 entries, found %d", len(retrievedEntries))
 	}
+	for _, dbEntry := range retrievedEntries {
+		if dbEntry.DeviceId != devId1 {
+			t.Fatalf("Response contains an incorrect device ID: %#v", *dbEntry)
+		}
+		if dbEntry.UserId != userId {
+			t.Fatalf("Response contains an incorrect user ID: %#v", *dbEntry)
+		}
+		if dbEntry.ReadCount != 1 {
+			t.Fatalf("db.ReadCount should have been 1, was %v", dbEntry.ReadCount)
+		}
+		decEntry, err := data.DecryptHistoryEntry("dkey", *dbEntry)
+		shared.Check(t, err)
+		if !data.EntryEquals(decEntry, entry1Dec) && !data.EntryEquals(decEntry, entry2Dec) {
+			t.Fatalf("DB data is different than input! \ndb   =%#v\nentry1=%#v\nentry2=%#v", *dbEntry, entry1Dec, entry2Dec)
+		}
+	}
 }
 
 func TestUpdateReleaseVersion(t *testing.T) {

client/lib/lib.go

@@ -198,12 +198,12 @@ func Setup(args []string) error {
 	db.Exec("DELETE FROM history_entries")
 
 	// Bootstrap from remote date
-	_, err = ApiGet("/api/v1/eregister?user_id=" + data.UserId(userSecret) + "&device_id=" + config.DeviceId)
+	_, err = ApiGet("/api/v1/register?user_id=" + data.UserId(userSecret) + "&device_id=" + config.DeviceId)
 	if err != nil {
 		return fmt.Errorf("failed to register device with backend: %v", err)
 	}
 
-	respBody, err := ApiGet("/api/v1/ebootstrap?user_id=" + data.UserId(userSecret) + "&device_id=" + config.DeviceId)
+	respBody, err := ApiGet("/api/v1/bootstrap?user_id=" + data.UserId(userSecret) + "&device_id=" + config.DeviceId)
 	if err != nil {
 		return fmt.Errorf("failed to bootstrap device from the backend: %v", err)
 	}

hishtory.go

@@ -60,7 +60,7 @@ func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error {
 	if err != nil {
 		return err
 	}
-	respBody, err := lib.ApiGet("/api/v1/equery?device_id=" + config.DeviceId + "&user_id=" + data.UserId(config.UserSecret))
+	respBody, err := lib.ApiGet("/api/v1/query?device_id=" + config.DeviceId + "&user_id=" + data.UserId(config.UserSecret))
 	if err != nil {
 		return err
 	}
@@ -131,10 +131,10 @@ func saveHistoryEntry() {
 	encEntry.DeviceId = config.DeviceId
 	jsonValue, err := json.Marshal([]shared.EncHistoryEntry{encEntry})
 	lib.CheckFatalError(err)
-	_, err = lib.ApiPost("/api/v1/esubmit", "application/json", jsonValue)
+	_, err = lib.ApiPost("/api/v1/submit", "application/json", jsonValue)
 	if err != nil {
 		if strings.Contains(err.Error(), "dial tcp: lookup api.hishtory.dev") {
-			// TODO: Somehow handle this
+			// TODO: Somehow handle this and don't completely lose it
 			lib.GetLogger().Printf("Failed to remotely persist hishtory entry because the device is offline!")
 		} else {
 			lib.CheckFatalError(err)