From feaa8b2bd1914ad440b7cac9d4848f32287199b2 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sun, 1 May 2022 22:37:26 -0400 Subject: [PATCH] Add a DB dump test that passes on zsh (is failing for an unknown reason on bash currently) + fix backup and restore for WAL files + better offline support --- backend/server/server.go | 12 ++++++ client/client_test.go | 90 ++++++++++++++++++++++++++++++++++++++++ client/data/data.go | 1 + client/lib/lib.go | 17 ++++++-- hishtory.go | 45 +++++++++++++++++--- shared/testutils.go | 34 ++++++++++----- 6 files changed, 181 insertions(+), 18 deletions(-) diff --git a/backend/server/server.go b/backend/server/server.go index 482d35c..82c7c8f 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -118,6 +118,8 @@ func apiQueryHandler(w http.ResponseWriter, r *http.Request) { w.Write(resp) } +// TODO: Add a helper to get query params and require them since most of these are meant to be mandatory + func apiRegisterHandler(w http.ResponseWriter, r *http.Request) { userId := r.URL.Query().Get("user_id") deviceId := r.URL.Query().Get("device_id") @@ -185,6 +187,13 @@ func apiBannerHandler(w http.ResponseWriter, r *http.Request) { w.Write([]byte(forcedBanner)) } +func wipeDbHandler(w http.ResponseWriter, r *http.Request) { + result := GLOBAL_DB.Exec("DELETE FROM enc_history_entries") + if result.Error != nil { + panic(result.Error) + } +} + func isTestEnvironment() bool { u, err := user.Current() if err != nil { @@ -443,5 +452,8 @@ func main() { http.Handle("/api/v1/banner", withLogging(apiBannerHandler)) http.Handle("/api/v1/download", withLogging(apiDownloadHandler)) http.Handle("/api/v1/trigger-cron", withLogging(triggerCronHandler)) + if isTestEnvironment() { + http.Handle("/api/v1/wipe-db", withLogging(wipeDbHandler)) + } log.Fatal(http.ListenAndServe(":8080", nil)) } diff --git a/client/client_test.go b/client/client_test.go index b098e9b..ffad34f 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -17,6 +17,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/ddworken/hishtory/client/data" + "github.com/ddworken/hishtory/client/lib" "github.com/ddworken/hishtory/shared" ) @@ -121,6 +122,7 @@ func TestParameterized(t *testing.T) { t.Run("testDisplayTable/"+tester.ShellName(), func(t *testing.T) { testDisplayTable(t, tester) }) t.Run("testTableDisplayCwd/"+tester.ShellName(), func(t *testing.T) { testTableDisplayCwd(t, tester) }) t.Run("testTimestampsAreReasonablyCorrect/"+tester.ShellName(), func(t *testing.T) { testTimestampsAreReasonablyCorrect(t, tester) }) + t.Run("testRequestAndReceiveDbDump/"+tester.ShellName(), func(t *testing.T) { testRequestAndReceiveDbDump(t, tester) }) } } @@ -893,3 +895,91 @@ func testDisplayTable(t *testing.T, tester shellTester) { t.Fatalf("hishtory query table test mismatch out=%#v", out) } } + +func testRequestAndReceiveDbDump(t *testing.T, tester shellTester) { + // Set up + defer shared.BackupAndRestore(t)() + secretKey := installHishtory(t, tester, "") + + // Record two commands and then query for them + out := tester.RunInteractiveShell(t, `echo hello +echo other +echo tododeleteme`) + // if out != "hello\nother\n" { + // t.Fatalf("running echo had unexpected out=%#v", out) + // } // todo + + // Query for it and check that the directory gets recorded correctly + time.Sleep(5 * time.Second) // todo: delete this + fmt.Println(tester.RunInteractiveShell(t, `hishtory export`)) // todo: delete + out = hishtoryQuery(t, tester, "echo") + if strings.Count(out, "\n") != 3 { + t.Fatalf("hishtory query has unexpected number of lines: out=%#v", out) + } + if !strings.Contains(out, "echo hello") { + t.Fatalf("hishtory query doesn't contain expected command, out=%#v", out) + } + if !strings.Contains(out, "echo other") { + t.Fatalf("hishtory query doesn't contain expected command, out=%#v", out) + } + + // Back up this copy + restoreFirstInstallation := shared.BackupAndRestoreWithId(t, "-install1") + + // Wipe the DB to simulate entries getting deleted because they've already been read and expired + _, err := lib.ApiGet("/api/v1/wipe-db") + if err != nil { + t.Fatalf("failed to wipe the DB: %v", err) + } + + // Install a new one (with the same secret key but a diff device id) + installHishtory(t, tester, secretKey) + + // Check that the new one doesn't have the commands yet + out = hishtoryQuery(t, tester, "echo") + if strings.Count(out, "\n") != 1 { + t.Fatalf("hishtory query has unexpected number of lines, should contain no entries: out=%#v", out) + } + if strings.Contains(out, "echo hello") || strings.Contains("echo other", out) { + t.Fatalf("hishtory query contains unexpected command, out=%#v", out) + } + out = tester.RunInteractiveShell(t, `hishtory export | grep -v pipefail`) + if out != "hishtory query echo\n" { + t.Fatalf("hishtory export has unexpected out=%#v", out) + } + + // Restore the first copy + restoreSecondInstallation := shared.BackupAndRestoreWithId(t, "-install2") + restoreFirstInstallation() + + // Confirm it still has the correct entries via hishtory export (and this runs a command to trigger it to dump the DB) + out = tester.RunInteractiveShell(t, `hishtory export | grep -v pipefail`) + if out != "echo hello\necho other\nhishtory query echo\nhishtory query echo\n" { + t.Fatalf("running hishtory export had unexpected out=%#v", out) + } + + // Restore the second copy and confirm it has the commands + restoreSecondInstallation() + fmt.Println(tester.RunInteractiveShell(t, `hishtory status -v`)) // todo: delete this + out = hishtoryQuery(t, tester, "ech") + if strings.Count(out, "\n") != 5 { + t.Fatalf("hishtory query has unexpected number of lines=%d: out=%#v", strings.Count(out, "\n"), out) + } + expected := []string{"echo hello", "echo other"} + for _, item := range expected { + if !strings.Contains(out, item) { + t.Fatalf("output is missing expected item %#v: %#v", item, out) + } + if strings.Count(out, item) != 1 { + t.Fatalf("output has %#v in it multiple times! out=%#v", item, out) + } + } + + // And check hishtory export too for good measure + out = tester.RunInteractiveShell(t, `hishtory export | grep -v pipefail`) + if out != "echo hello\necho other\nhishtory query echo\nhishtory query echo\nhishtory query ech\n" { + t.Fatalf("running hishtory export had unexpected out=%#v", out) + } +} + +// TODO: write a test that runs hishtroy export | grep -v pipefail and then see if that shows up in query/export, I think there is weird behavior here diff --git a/client/data/data.go b/client/data/data.go index 0a0b3c7..90f8602 100644 --- a/client/data/data.go +++ b/client/data/data.go @@ -33,6 +33,7 @@ type HistoryEntry struct { ExitCode int `json:"exit_code" gorm:"uniqueIndex:compositeindex"` StartTime time.Time `json:"start_time" gorm:"uniqueIndex:compositeindex"` EndTime time.Time `json:"end_time" gorm:"uniqueIndex:compositeindex"` + DeviceId string `json:"device_id" gorm:"uniqueIndex:compositeindex"` } func sha256hmac(key, additionalData string) []byte { diff --git a/client/lib/lib.go b/client/lib/lib.go index 43c2de8..17b06ee 100644 --- a/client/lib/lib.go +++ b/client/lib/lib.go @@ -137,6 +137,13 @@ func BuildHistoryEntry(args []string) (*data.HistoryEntry, error) { } entry.Hostname = hostname + // device ID + config, err := GetConfig() + if err != nil { + return nil, fmt.Errorf("failed to get device ID when building history entry: %v", err) + } + entry.DeviceId = config.DeviceId + return &entry, nil } @@ -686,15 +693,15 @@ func ApiGet(path string) ([]byte, error) { start := time.Now() resp, err := http.Get(getServerHostname() + path) if err != nil { - return nil, fmt.Errorf("failed to GET %s: %v", path, err) + return nil, fmt.Errorf("failed to GET %s%s: %v", getServerHostname(), path, err) } defer resp.Body.Close() if resp.StatusCode != 200 { - return nil, fmt.Errorf("failed to GET %s: status_code=%d", path, resp.StatusCode) + return nil, fmt.Errorf("failed to GET %s%s: status_code=%d", getServerHostname(), path, resp.StatusCode) } respBody, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read response body from GET %s: %v", path, err) + return nil, fmt.Errorf("failed to read response body from GET %s%s: %v", getServerHostname(), path, err) } duration := time.Since(start) GetLogger().Printf("ApiGet(%#v): %s\n", path, duration.String()) @@ -796,3 +803,7 @@ func setCodesigningXattrs(downloadInfo shared.UpdateInfo, filename string) error setXattr(filename, string(xattrDump)) return nil } + +func IsOfflineError(err error) bool { + return strings.Contains(err.Error(), "dial tcp: lookup api.hishtory.dev") || strings.Contains(err.Error(), "read: connection reset by peer") +} diff --git a/hishtory.go b/hishtory.go index 23362d7..16ba453 100644 --- a/hishtory.go +++ b/hishtory.go @@ -46,6 +46,7 @@ func main() { if len(os.Args) == 3 && os.Args[2] == "-v" { fmt.Printf("User ID: %s\n", data.UserId(config.UserSecret)) fmt.Printf("Device ID: %s\n", config.DeviceId) + printDumpStatus(config) } fmt.Printf("Commit Hash: %s\n", GitCommit) case "update": @@ -55,6 +56,17 @@ func main() { } } +func printDumpStatus(config lib.ClientConfig) { + respBody, err := lib.ApiGet("/api/v1/get-dump-requests?user_id=" + data.UserId(config.UserSecret)) + if err != nil { + lib.CheckFatalError(err) + } + var dumpRequests []*shared.DumpRequest + err = json.Unmarshal(respBody, &dumpRequests) + lib.CheckFatalError(err) + fmt.Printf("Dump Requests: %#v\n", dumpRequests) +} + func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error { config, err := lib.GetConfig() if err != nil { @@ -69,11 +81,13 @@ func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error { if err != nil { return fmt.Errorf("failed to load JSON response: %v", err) } + // fmt.Printf("this device id=%s, user id=%s\n", config.DeviceId, data.UserId(config.UserSecret)) for _, entry := range retrievedEntries { decEntry, err := data.DecryptHistoryEntry(config.UserSecret, *entry) if err != nil { return fmt.Errorf("failed to decrypt history entry from server: %v", err) } + // fmt.Printf("received entry: %#v\n", decEntry) lib.AddToDbIfNew(db, decEntry) } return nil @@ -82,7 +96,14 @@ func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error { func query(query string) { db, err := lib.OpenLocalSqliteDb() lib.CheckFatalError(err) - lib.CheckFatalError(retrieveAdditionalEntriesFromRemote(db)) + err = retrieveAdditionalEntriesFromRemote(db) + if err != nil { + if lib.IsOfflineError(err) { + fmt.Println("Warning: hishtory is offline so this may be missing recent results from your other machines!") + } else { + lib.CheckFatalError(err) + } + } lib.CheckFatalError(displayBannerIfSet()) data, err := data.Search(db, query, 25) lib.CheckFatalError(err) @@ -133,7 +154,7 @@ func saveHistoryEntry() { lib.CheckFatalError(err) _, err = lib.ApiPost("/api/v1/submit", "application/json", jsonValue) if err != nil { - if strings.Contains(err.Error(), "dial tcp: lookup api.hishtory.dev") || strings.Contains(err.Error(), "read: connection reset by peer") { + if lib.IsOfflineError(err) { // TODO: Somehow handle this and don't completely lose it lib.GetLogger().Printf("Failed to remotely persist hishtory entry because the device is offline!") } else { @@ -143,7 +164,14 @@ func saveHistoryEntry() { // Check if there is a pending dump request and reply to it if so resp, err := lib.ApiGet("/api/v1/get-dump-requests?user_id=" + data.UserId(config.UserSecret) + "&device_id=" + config.DeviceId) - lib.CheckFatalError(err) + if err != nil { + if lib.IsOfflineError(err) { + // It is fine to just ignore this, the next command will retry the API and eventually we will respond to any pending dump requests + resp = []byte("[]") + } else { + lib.CheckFatalError(err) + } + } var dumpRequests []*shared.DumpRequest err = json.Unmarshal(resp, &dumpRequests) lib.CheckFatalError(err) @@ -160,7 +188,7 @@ func saveHistoryEntry() { reqBody, err := json.Marshal(encEntries) lib.CheckFatalError(err) for _, dumpRequest := range dumpRequests { - _, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId, "application/json", reqBody) + _, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId+"&source_device_id="+config.DeviceId, "application/json", reqBody) lib.CheckFatalError(err) } } @@ -169,7 +197,14 @@ func saveHistoryEntry() { func export() { db, err := lib.OpenLocalSqliteDb() lib.CheckFatalError(err) - lib.CheckFatalError(retrieveAdditionalEntriesFromRemote(db)) + err = retrieveAdditionalEntriesFromRemote(db) + if err != nil { + if lib.IsOfflineError(err) { + fmt.Println("Warning: hishtory is offline so this may be missing recent results from your other machines!") + } else { + lib.CheckFatalError(err) + } + } data, err := data.Search(db, "", 0) lib.CheckFatalError(err) for i := len(data) - 1; i >= 0; i-- { diff --git a/shared/testutils.go b/shared/testutils.go index b597f7c..c6c799f 100644 --- a/shared/testutils.go +++ b/shared/testutils.go @@ -13,6 +13,11 @@ import ( "time" ) +const ( + DB_WAL_PATH = DB_PATH + "-wal" + DB_SHM_PATH = DB_PATH + "-shm" +) + func ResetLocalState(t *testing.T) { homedir, err := os.UserHomeDir() if err != nil { @@ -20,6 +25,7 @@ func ResetLocalState(t *testing.T) { } _ = os.Remove(path.Join(homedir, HISHTORY_PATH, DB_PATH)) + _ = os.Remove(path.Join(homedir, HISHTORY_PATH, DB_WAL_PATH)) _ = os.Remove(path.Join(homedir, HISHTORY_PATH, CONFIG_PATH)) _ = os.Remove(path.Join(homedir, HISHTORY_PATH, "hishtory")) _ = os.Remove(path.Join(homedir, HISHTORY_PATH, "config.sh")) @@ -27,22 +33,30 @@ func ResetLocalState(t *testing.T) { } func BackupAndRestore(t *testing.T) func() { + return BackupAndRestoreWithId(t, "") +} + +func BackupAndRestoreWithId(t *testing.T, id string) func() { homedir, err := os.UserHomeDir() if err != nil { t.Fatalf("failed to retrieve homedir: %v", err) } - _ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_PATH), path.Join(homedir, HISHTORY_PATH, DB_PATH+".bak")) - _ = os.Rename(path.Join(homedir, HISHTORY_PATH, CONFIG_PATH), path.Join(homedir, HISHTORY_PATH, CONFIG_PATH+".bak")) - _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "hishtory"), path.Join(homedir, HISHTORY_PATH, "hishtory.bak")) - _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.sh"), path.Join(homedir, HISHTORY_PATH, "config.sh.bak")) - _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.zsh"), path.Join(homedir, HISHTORY_PATH, "config.zsh.bak")) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_PATH), path.Join(homedir, HISHTORY_PATH, DB_PATH+id+".bak")) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_WAL_PATH), path.Join(homedir, HISHTORY_PATH, DB_WAL_PATH+id+".bak")) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_SHM_PATH), path.Join(homedir, HISHTORY_PATH, DB_SHM_PATH+id+".bak")) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, CONFIG_PATH), path.Join(homedir, HISHTORY_PATH, CONFIG_PATH+id+".bak")) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "hishtory"), path.Join(homedir, HISHTORY_PATH, "hishtory"+id+".bak")) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.sh"), path.Join(homedir, HISHTORY_PATH, "config.sh"+id+"bak")) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.zsh"), path.Join(homedir, HISHTORY_PATH, "config.zsh"+id+".bak")) return func() { - _ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_PATH+".bak"), path.Join(homedir, HISHTORY_PATH, DB_PATH)) - _ = os.Rename(path.Join(homedir, HISHTORY_PATH, CONFIG_PATH+".bak"), path.Join(homedir, HISHTORY_PATH, CONFIG_PATH)) - _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "hishtory.bak"), path.Join(homedir, HISHTORY_PATH, "hishtory")) - _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.sh.bak"), path.Join(homedir, HISHTORY_PATH, "config.sh")) - _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.zsh.bak"), path.Join(homedir, HISHTORY_PATH, "config.zsh")) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_PATH+id+".bak"), path.Join(homedir, HISHTORY_PATH, DB_PATH)) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_WAL_PATH+id+".bak"), path.Join(homedir, HISHTORY_PATH, DB_WAL_PATH)) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_SHM_PATH+id+".bak"), path.Join(homedir, HISHTORY_PATH, DB_SHM_PATH)) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, CONFIG_PATH+id+".bak"), path.Join(homedir, HISHTORY_PATH, CONFIG_PATH)) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "hishtory"+id+".bak"), path.Join(homedir, HISHTORY_PATH, "hishtory")) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.sh"+id+".bak"), path.Join(homedir, HISHTORY_PATH, "config.sh")) + _ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.zsh"+id+".bak"), path.Join(homedir, HISHTORY_PATH, "config.zsh")) } }