Remove the 'e' prefix from api endpoints + implement backend APIs for clean loading of all data from other instances

This commit is contained in:
David Dworken 2022-04-28 10:56:59 -07:00
parent 87c8c961e9
commit 0499010981
4 changed files with 230 additions and 44 deletions

View File

@ -7,6 +7,7 @@ import (
"log"
"net/http"
"os"
"os/user"
"strconv"
"strings"
"time"
@ -45,7 +46,13 @@ func updateUsageData(userId, deviceId string) {
}
}
func apiESubmitHandler(w http.ResponseWriter, r *http.Request) {
type DumpRequest struct {
UserId string `json:"user_id"`
RequestingDeviceId string `json:"requesting_device_id"`
RequestTime time.Time `json:"request_time"`
}
func apiSubmitHandler(w http.ResponseWriter, r *http.Request) {
data, err := ioutil.ReadAll(r.Body)
if err != nil {
panic(err)
@ -55,7 +62,7 @@ func apiESubmitHandler(w http.ResponseWriter, r *http.Request) {
if err != nil {
panic(fmt.Sprintf("body=%#v, err=%v", data, err))
}
fmt.Printf("apiESubmitHandler: received request containg %d EncHistoryEntry\n", len(entries))
fmt.Printf("apiSubmitHandler: received request containg %d EncHistoryEntry\n", len(entries))
for _, entry := range entries {
updateUsageData(entry.UserId, entry.DeviceId)
tx := GLOBAL_DB.Where("user_id = ?", entry.UserId)
@ -67,7 +74,7 @@ func apiESubmitHandler(w http.ResponseWriter, r *http.Request) {
if len(devices) == 0 {
panic(fmt.Errorf("found no devices associated with user_id=%s, can't save history entry", entry.UserId))
}
fmt.Printf("apiESubmitHandler: Found %d devices\n", len(devices))
fmt.Printf("apiSubmitHandler: Found %d devices\n", len(devices))
for _, device := range devices {
entry.DeviceId = device.DeviceId
result := GLOBAL_DB.Create(&entry)
@ -78,7 +85,7 @@ func apiESubmitHandler(w http.ResponseWriter, r *http.Request) {
}
}
func apiEQueryHandler(w http.ResponseWriter, r *http.Request) {
func apiQueryHandler(w http.ResponseWriter, r *http.Request) {
userId := r.URL.Query().Get("user_id")
deviceId := r.URL.Query().Get("device_id")
updateUsageData(userId, deviceId)
@ -92,7 +99,7 @@ func apiEQueryHandler(w http.ResponseWriter, r *http.Request) {
if result.Error != nil {
panic(fmt.Errorf("DB query error: %v", result.Error))
}
fmt.Printf("apiEQueryHandler: Found %d entries\n", len(historyEntries))
fmt.Printf("apiQueryHandler: Found %d entries\n", len(historyEntries))
resp, err := json.Marshal(historyEntries)
if err != nil {
panic(err)
@ -100,31 +107,63 @@ func apiEQueryHandler(w http.ResponseWriter, r *http.Request) {
w.Write(resp)
}
// TODO: bootstrap is a janky solution for the initial version of this. Long term, need to support deleting entries from the DB which means replacing bootstrap with a queued message sent to any live instances.
func apiEBootstrapHandler(w http.ResponseWriter, r *http.Request) {
userId := r.URL.Query().Get("user_id")
deviceId := r.URL.Query().Get("device_id")
updateUsageData(userId, deviceId)
tx := GLOBAL_DB.Where("user_id = ?", userId)
var historyEntries []*shared.EncHistoryEntry
result := tx.Find(&historyEntries)
if result.Error != nil {
panic(fmt.Errorf("DB query error: %v", result.Error))
}
resp, err := json.Marshal(historyEntries)
if err != nil {
panic(err)
}
w.Write(resp)
}
func apiERegisterHandler(w http.ResponseWriter, r *http.Request) {
func apiRegisterHandler(w http.ResponseWriter, r *http.Request) {
userId := r.URL.Query().Get("user_id")
deviceId := r.URL.Query().Get("device_id")
GLOBAL_DB.Create(&shared.Device{UserId: userId, DeviceId: deviceId, RegistrationIp: r.RemoteAddr, RegistrationDate: time.Now()})
GLOBAL_DB.Create(&DumpRequest{UserId: userId, RequestingDeviceId: deviceId, RequestTime: time.Now()})
updateUsageData(userId, deviceId)
}
func apiGetPendingDumpRequestsHandler(w http.ResponseWriter, r *http.Request) {
userId := r.URL.Query().Get("user_id")
var dumpRequests []*DumpRequest
result := GLOBAL_DB.Where("user_id = ?", userId).Find(&dumpRequests)
if result.Error != nil {
panic(fmt.Errorf("DB query error: %v", result.Error))
}
respBody, err := json.Marshal(dumpRequests)
if err != nil {
panic(fmt.Errorf("failed to JSON marshall the dump requests: %v", err))
}
w.Write(respBody)
}
func apiSubmitDumpHandler(w http.ResponseWriter, r *http.Request) {
userId := r.URL.Query().Get("user_id")
requestingDeviceId := r.URL.Query().Get("requesting_device_id")
data, err := ioutil.ReadAll(r.Body)
if err != nil {
panic(err)
}
var entries []shared.EncHistoryEntry
err = json.Unmarshal(data, &entries)
if err != nil {
panic(fmt.Sprintf("body=%#v, err=%v", data, err))
}
fmt.Printf("apiSubmitDumpHandler: received request containg %d EncHistoryEntry\n", len(entries))
err = GLOBAL_DB.Transaction(func(tx *gorm.DB) error {
for _, entry := range entries {
entry.DeviceId = requestingDeviceId
if entry.UserId != userId {
return fmt.Errorf("batch contains an entry with UserId=%#v, when the query param contained the user_id=%#v", entry.UserId, userId)
}
result := tx.Create(&entry)
if result.Error != nil {
return fmt.Errorf("failed to create entry: %v", err)
}
}
return nil
})
if err != nil {
panic(fmt.Errorf("failed to execute transaction to add dumped DB: %v", err))
}
result := GLOBAL_DB.Delete(&DumpRequest{}, "user_id = ? AND requesting_device_id = ?", userId, requestingDeviceId)
if result.Error != nil {
panic(fmt.Errorf("failed to clear the dump request: %v", err))
}
}
func apiBannerHandler(w http.ResponseWriter, r *http.Request) {
commitHash := r.URL.Query().Get("commit_hash")
deviceId := r.URL.Query().Get("device_id")
@ -134,7 +173,11 @@ func apiBannerHandler(w http.ResponseWriter, r *http.Request) {
}
func isTestEnvironment() bool {
return os.Getenv("HISHTORY_TEST") != ""
u, err := user.Current()
if err != nil {
panic(err)
}
return os.Getenv("HISHTORY_TEST") != "" || u.Username == "david"
}
func OpenDB() (*gorm.DB, error) {
@ -146,6 +189,8 @@ func OpenDB() (*gorm.DB, error) {
db.AutoMigrate(&shared.EncHistoryEntry{})
db.AutoMigrate(&shared.Device{})
db.AutoMigrate(&UsageData{})
db.AutoMigrate(&DumpRequest{})
db.Exec("PRAGMA journal_mode = WAL")
return db, nil
}
@ -156,6 +201,7 @@ func OpenDB() (*gorm.DB, error) {
db.AutoMigrate(&shared.EncHistoryEntry{})
db.AutoMigrate(&shared.Device{})
db.AutoMigrate(&UsageData{})
db.AutoMigrate(&DumpRequest{})
return db, nil
}
@ -180,6 +226,7 @@ func cron() error {
}
func runBackgroundJobs() {
time.Sleep(5 * time.Second)
for {
err := cron()
if err != nil {
@ -374,12 +421,13 @@ func cleanDatabase() error {
func main() {
fmt.Println("Listening on localhost:8080")
http.Handle("/api/v1/esubmit", withLogging(apiESubmitHandler))
http.Handle("/api/v1/equery", withLogging(apiEQueryHandler))
http.Handle("/api/v1/ebootstrap", withLogging(apiEBootstrapHandler))
http.Handle("/api/v1/eregister", withLogging(apiERegisterHandler))
http.Handle("/api/v1/submit", withLogging(apiSubmitHandler))
http.Handle("/api/v1/get-dump-requests", withLogging(apiGetPendingDumpRequestsHandler))
http.Handle("/api/v1/submit-dump", withLogging(apiSubmitDumpHandler))
http.Handle("/api/v1/query", withLogging(apiQueryHandler))
http.Handle("/api/v1/register", withLogging(apiRegisterHandler))
http.Handle("/api/v1/banner", withLogging(apiBannerHandler))
http.Handle("/api/v1/trigger-cron", withLogging(triggerCronHandler))
http.Handle("/api/v1/download", withLogging(apiDownloadHandler))
http.Handle("/api/v1/trigger-cron", withLogging(triggerCronHandler))
log.Fatal(http.ListenAndServe(":8080", nil))
}

View File

@ -26,11 +26,11 @@ func TestESubmitThenQuery(t *testing.T) {
otherUser := data.UserId("otherkey")
otherDev := uuid.Must(uuid.NewRandom()).String()
deviceReq := httptest.NewRequest(http.MethodGet, "/?device_id="+devId1+"&user_id="+userId, nil)
apiERegisterHandler(nil, deviceReq)
apiRegisterHandler(nil, deviceReq)
deviceReq = httptest.NewRequest(http.MethodGet, "/?device_id="+devId2+"&user_id="+userId, nil)
apiERegisterHandler(nil, deviceReq)
apiRegisterHandler(nil, deviceReq)
deviceReq = httptest.NewRequest(http.MethodGet, "/?device_id="+otherDev+"&user_id="+otherUser, nil)
apiERegisterHandler(nil, deviceReq)
apiRegisterHandler(nil, deviceReq)
// Submit a few entries for different devices
entry := data.MakeFakeHistoryEntry("ls ~/")
@ -39,12 +39,12 @@ func TestESubmitThenQuery(t *testing.T) {
reqBody, err := json.Marshal([]shared.EncHistoryEntry{encEntry})
shared.Check(t, err)
submitReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(reqBody))
apiESubmitHandler(nil, submitReq)
apiSubmitHandler(nil, submitReq)
// Query for device id 1
w := httptest.NewRecorder()
searchReq := httptest.NewRequest(http.MethodGet, "/?device_id="+devId1+"&user_id="+userId, nil)
apiEQueryHandler(w, searchReq)
apiQueryHandler(w, searchReq)
res := w.Result()
defer res.Body.Close()
respBody, err := ioutil.ReadAll(res.Body)
@ -73,7 +73,7 @@ func TestESubmitThenQuery(t *testing.T) {
// Same for device id 2
w = httptest.NewRecorder()
searchReq = httptest.NewRequest(http.MethodGet, "/?device_id="+devId2+"&user_id="+userId, nil)
apiEQueryHandler(w, searchReq)
apiQueryHandler(w, searchReq)
res = w.Result()
defer res.Body.Close()
respBody, err = ioutil.ReadAll(res.Body)
@ -97,19 +97,157 @@ func TestESubmitThenQuery(t *testing.T) {
if !data.EntryEquals(decEntry, entry) {
t.Fatalf("DB data is different than input! \ndb =%#v\ninput=%#v", *dbEntry, entry)
}
}
// Bootstrap handler should return 2 entries, one for each device
func TestDumpRequestAndResponse(t *testing.T) {
// Set up
defer shared.BackupAndRestore(t)()
InitDB()
// Register a first device for two different users
userId := data.UserId("dkey")
devId1 := uuid.Must(uuid.NewRandom()).String()
otherUser := data.UserId("dOtherkey")
otherDev1 := uuid.Must(uuid.NewRandom()).String()
deviceReq := httptest.NewRequest(http.MethodGet, "/?device_id="+devId1+"&user_id="+userId, nil)
apiRegisterHandler(nil, deviceReq)
deviceReq = httptest.NewRequest(http.MethodGet, "/?device_id="+otherDev1+"&user_id="+otherUser, nil)
apiRegisterHandler(nil, deviceReq)
// Query for dump requests, there should be one for userId
w := httptest.NewRecorder()
apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/?user_id="+userId, nil))
res := w.Result()
defer res.Body.Close()
respBody, err := ioutil.ReadAll(res.Body)
shared.Check(t, err)
var dumpRequests []*DumpRequest
shared.Check(t, json.Unmarshal(respBody, &dumpRequests))
if len(dumpRequests) != 1 {
t.Fatalf("expected one pending dump request, got %#v", dumpRequests)
}
dumpRequest := dumpRequests[0]
if dumpRequest.RequestingDeviceId != devId1 {
t.Fatalf("unexpected device ID")
}
if dumpRequest.UserId != userId {
t.Fatalf("unexpected user ID")
}
// And one for otherUser
w = httptest.NewRecorder()
searchReq = httptest.NewRequest(http.MethodGet, "/?user_id="+data.UserId("key")+"&device_id="+devId1, nil)
apiEBootstrapHandler(w, searchReq)
apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/?user_id="+otherUser, nil))
res = w.Result()
defer res.Body.Close()
respBody, err = ioutil.ReadAll(res.Body)
shared.Check(t, err)
dumpRequests = make([]*DumpRequest, 0)
shared.Check(t, json.Unmarshal(respBody, &dumpRequests))
if len(dumpRequests) != 1 {
t.Fatalf("expected one pending dump request, got %#v", dumpRequests)
}
dumpRequest = dumpRequests[0]
if dumpRequest.RequestingDeviceId != otherDev1 {
t.Fatalf("unexpected device ID")
}
if dumpRequest.UserId != otherUser {
t.Fatalf("unexpected user ID")
}
// And none if we query without a user ID
w = httptest.NewRecorder()
apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/", nil))
res = w.Result()
defer res.Body.Close()
respBody, err = ioutil.ReadAll(res.Body)
shared.Check(t, err)
if string(respBody) != "[]" {
t.Fatalf("got unexpected respBody: %#v", string(respBody))
}
// And none if we query for a user ID that doesn't exit
w = httptest.NewRecorder()
apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/?user_id=foo", nil))
res = w.Result()
defer res.Body.Close()
respBody, err = ioutil.ReadAll(res.Body)
shared.Check(t, err)
if string(respBody) != "[]" {
t.Fatalf("got unexpected respBody: %#v", string(respBody))
}
// Now submit a dump for userId
entry1Dec := data.MakeFakeHistoryEntry("ls ~/")
entry1, err := data.EncryptHistoryEntry("dkey", entry1Dec)
shared.Check(t, err)
entry2Dec := data.MakeFakeHistoryEntry("aaaaaaáaaa")
entry2, err := data.EncryptHistoryEntry("dkey", entry1Dec)
shared.Check(t, err)
reqBody, err := json.Marshal([]shared.EncHistoryEntry{entry1, entry2})
shared.Check(t, err)
submitReq := httptest.NewRequest(http.MethodPost, "/?user_id="+userId+"&requesting_device_id="+devId1, bytes.NewReader(reqBody))
apiSubmitDumpHandler(nil, submitReq)
// Check that the dump request is no longer there for userId
w = httptest.NewRecorder()
apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/?user_id="+userId, nil))
res = w.Result()
defer res.Body.Close()
respBody, err = ioutil.ReadAll(res.Body)
shared.Check(t, err)
if string(respBody) != "[]" {
t.Fatalf("got unexpected respBody: %#v", string(respBody))
}
// But it is there for the other user
w = httptest.NewRecorder()
apiGetPendingDumpRequestsHandler(w, httptest.NewRequest(http.MethodGet, "/?user_id="+otherUser, nil))
res = w.Result()
defer res.Body.Close()
respBody, err = ioutil.ReadAll(res.Body)
shared.Check(t, err)
dumpRequests = make([]*DumpRequest, 0)
shared.Check(t, json.Unmarshal(respBody, &dumpRequests))
if len(dumpRequests) != 1 {
t.Fatalf("expected one pending dump request, got %#v", dumpRequests)
}
dumpRequest = dumpRequests[0]
if dumpRequest.RequestingDeviceId != otherDev1 {
t.Fatalf("unexpected device ID")
}
if dumpRequest.UserId != otherUser {
t.Fatalf("unexpected user ID")
}
// And finally, query to ensure that the dumped entries are in the DB
w = httptest.NewRecorder()
searchReq := httptest.NewRequest(http.MethodGet, "/?device_id="+devId1+"&user_id="+userId, nil)
apiQueryHandler(w, searchReq)
res = w.Result()
defer res.Body.Close()
respBody, err = ioutil.ReadAll(res.Body)
shared.Check(t, err)
var retrievedEntries []*shared.EncHistoryEntry
shared.Check(t, json.Unmarshal(respBody, &retrievedEntries))
if len(retrievedEntries) != 2 {
t.Fatalf("Expected to retrieve 2 entries, found %d", len(retrievedEntries))
}
for _, dbEntry := range retrievedEntries {
if dbEntry.DeviceId != devId1 {
t.Fatalf("Response contains an incorrect device ID: %#v", *dbEntry)
}
if dbEntry.UserId != userId {
t.Fatalf("Response contains an incorrect user ID: %#v", *dbEntry)
}
if dbEntry.ReadCount != 1 {
t.Fatalf("db.ReadCount should have been 1, was %v", dbEntry.ReadCount)
}
decEntry, err := data.DecryptHistoryEntry("dkey", *dbEntry)
shared.Check(t, err)
if !data.EntryEquals(decEntry, entry1Dec) && !data.EntryEquals(decEntry, entry2Dec) {
t.Fatalf("DB data is different than input! \ndb =%#v\nentry1=%#v\nentry2=%#v", *dbEntry, entry1Dec, entry2Dec)
}
}
}
func TestUpdateReleaseVersion(t *testing.T) {

View File

@ -198,12 +198,12 @@ func Setup(args []string) error {
db.Exec("DELETE FROM history_entries")
// Bootstrap from remote date
_, err = ApiGet("/api/v1/eregister?user_id=" + data.UserId(userSecret) + "&device_id=" + config.DeviceId)
_, err = ApiGet("/api/v1/register?user_id=" + data.UserId(userSecret) + "&device_id=" + config.DeviceId)
if err != nil {
return fmt.Errorf("failed to register device with backend: %v", err)
}
respBody, err := ApiGet("/api/v1/ebootstrap?user_id=" + data.UserId(userSecret) + "&device_id=" + config.DeviceId)
respBody, err := ApiGet("/api/v1/bootstrap?user_id=" + data.UserId(userSecret) + "&device_id=" + config.DeviceId)
if err != nil {
return fmt.Errorf("failed to bootstrap device from the backend: %v", err)
}

View File

@ -60,7 +60,7 @@ func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error {
if err != nil {
return err
}
respBody, err := lib.ApiGet("/api/v1/equery?device_id=" + config.DeviceId + "&user_id=" + data.UserId(config.UserSecret))
respBody, err := lib.ApiGet("/api/v1/query?device_id=" + config.DeviceId + "&user_id=" + data.UserId(config.UserSecret))
if err != nil {
return err
}
@ -131,10 +131,10 @@ func saveHistoryEntry() {
encEntry.DeviceId = config.DeviceId
jsonValue, err := json.Marshal([]shared.EncHistoryEntry{encEntry})
lib.CheckFatalError(err)
_, err = lib.ApiPost("/api/v1/esubmit", "application/json", jsonValue)
_, err = lib.ApiPost("/api/v1/submit", "application/json", jsonValue)
if err != nil {
if strings.Contains(err.Error(), "dial tcp: lookup api.hishtory.dev") {
// TODO: Somehow handle this
// TODO: Somehow handle this and don't completely lose it
lib.GetLogger().Printf("Failed to remotely persist hishtory entry because the device is offline!")
} else {
lib.CheckFatalError(err)