Add a DB dump test that passes on zsh (is failing for an unknown reason on bash currently) + fix backup and restore for WAL files + better offline support

backend/server/server.go

@@ -118,6 +118,8 @@ func apiQueryHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write(resp)
 }
 
+// TODO: Add a helper to get query params and require them since most of these are meant to be mandatory
+
 func apiRegisterHandler(w http.ResponseWriter, r *http.Request) {
 	userId := r.URL.Query().Get("user_id")
 	deviceId := r.URL.Query().Get("device_id")
@@ -185,6 +187,13 @@ func apiBannerHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(forcedBanner))
 }
 
+func wipeDbHandler(w http.ResponseWriter, r *http.Request) {
+	result := GLOBAL_DB.Exec("DELETE FROM enc_history_entries")
+	if result.Error != nil {
+		panic(result.Error)
+	}
+}
+
 func isTestEnvironment() bool {
 	u, err := user.Current()
 	if err != nil {
@@ -443,5 +452,8 @@ func main() {
 	http.Handle("/api/v1/banner", withLogging(apiBannerHandler))
 	http.Handle("/api/v1/download", withLogging(apiDownloadHandler))
 	http.Handle("/api/v1/trigger-cron", withLogging(triggerCronHandler))
+	if isTestEnvironment() {
+		http.Handle("/api/v1/wipe-db", withLogging(wipeDbHandler))
+	}
 	log.Fatal(http.ListenAndServe(":8080", nil))
 }

client/client_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 
 	"github.com/ddworken/hishtory/client/data"
+	"github.com/ddworken/hishtory/client/lib"
 	"github.com/ddworken/hishtory/shared"
 )
 
@@ -121,6 +122,7 @@ func TestParameterized(t *testing.T) {
 		t.Run("testDisplayTable/"+tester.ShellName(), func(t *testing.T) { testDisplayTable(t, tester) })
 		t.Run("testTableDisplayCwd/"+tester.ShellName(), func(t *testing.T) { testTableDisplayCwd(t, tester) })
 		t.Run("testTimestampsAreReasonablyCorrect/"+tester.ShellName(), func(t *testing.T) { testTimestampsAreReasonablyCorrect(t, tester) })
+		t.Run("testRequestAndReceiveDbDump/"+tester.ShellName(), func(t *testing.T) { testRequestAndReceiveDbDump(t, tester) })
 	}
 }
 
@@ -893,3 +895,91 @@ func testDisplayTable(t *testing.T, tester shellTester) {
 		t.Fatalf("hishtory query table test mismatch out=%#v", out)
 	}
 }
+
+func testRequestAndReceiveDbDump(t *testing.T, tester shellTester) {
+	// Set up
+	defer shared.BackupAndRestore(t)()
+	secretKey := installHishtory(t, tester, "")
+
+	// Record two commands and then query for them
+	out := tester.RunInteractiveShell(t, `echo hello
+echo other
+echo tododeleteme`)
+	// if out != "hello\nother\n" {
+	// 	t.Fatalf("running echo had unexpected out=%#v", out)
+	// } // todo
+
+	// Query for it and check that the directory gets recorded correctly
+	time.Sleep(5 * time.Second)                                   // todo: delete this
+	fmt.Println(tester.RunInteractiveShell(t, `hishtory export`)) // todo: delete
+	out = hishtoryQuery(t, tester, "echo")
+	if strings.Count(out, "\n") != 3 {
+		t.Fatalf("hishtory query has unexpected number of lines: out=%#v", out)
+	}
+	if !strings.Contains(out, "echo hello") {
+		t.Fatalf("hishtory query doesn't contain expected command, out=%#v", out)
+	}
+	if !strings.Contains(out, "echo other") {
+		t.Fatalf("hishtory query doesn't contain expected command, out=%#v", out)
+	}
+
+	// Back up this copy
+	restoreFirstInstallation := shared.BackupAndRestoreWithId(t, "-install1")
+
+	// Wipe the DB to simulate entries getting deleted because they've already been read and expired
+	_, err := lib.ApiGet("/api/v1/wipe-db")
+	if err != nil {
+		t.Fatalf("failed to wipe the DB: %v", err)
+	}
+
+	// Install a new one (with the same secret key but a diff device id)
+	installHishtory(t, tester, secretKey)
+
+	// Check that the new one doesn't have the commands yet
+	out = hishtoryQuery(t, tester, "echo")
+	if strings.Count(out, "\n") != 1 {
+		t.Fatalf("hishtory query has unexpected number of lines, should contain no entries: out=%#v", out)
+	}
+	if strings.Contains(out, "echo hello") || strings.Contains("echo other", out) {
+		t.Fatalf("hishtory query contains unexpected command, out=%#v", out)
+	}
+	out = tester.RunInteractiveShell(t, `hishtory export | grep -v pipefail`)
+	if out != "hishtory query echo\n" {
+		t.Fatalf("hishtory export has unexpected out=%#v", out)
+	}
+
+	// Restore the first copy
+	restoreSecondInstallation := shared.BackupAndRestoreWithId(t, "-install2")
+	restoreFirstInstallation()
+
+	// Confirm it still has the correct entries via hishtory export (and this runs a command to trigger it to dump the DB)
+	out = tester.RunInteractiveShell(t, `hishtory export | grep -v pipefail`)
+	if out != "echo hello\necho other\nhishtory query echo\nhishtory query echo\n" {
+		t.Fatalf("running hishtory export had unexpected out=%#v", out)
+	}
+
+	// Restore the second copy and confirm it has the commands
+	restoreSecondInstallation()
+	fmt.Println(tester.RunInteractiveShell(t, `hishtory status -v`)) // todo: delete this
+	out = hishtoryQuery(t, tester, "ech")
+	if strings.Count(out, "\n") != 5 {
+		t.Fatalf("hishtory query has unexpected number of lines=%d: out=%#v", strings.Count(out, "\n"), out)
+	}
+	expected := []string{"echo hello", "echo other"}
+	for _, item := range expected {
+		if !strings.Contains(out, item) {
+			t.Fatalf("output is missing expected item %#v: %#v", item, out)
+		}
+		if strings.Count(out, item) != 1 {
+			t.Fatalf("output has %#v in it multiple times! out=%#v", item, out)
+		}
+	}
+
+	// And check hishtory export too for good measure
+	out = tester.RunInteractiveShell(t, `hishtory export | grep -v pipefail`)
+	if out != "echo hello\necho other\nhishtory query echo\nhishtory query echo\nhishtory query ech\n" {
+		t.Fatalf("running hishtory export had unexpected out=%#v", out)
+	}
+}
+
+// TODO: write a test that runs hishtroy export | grep -v pipefail and then see if that shows up in query/export, I think there is weird behavior here

client/data/data.go

@@ -33,6 +33,7 @@ type HistoryEntry struct {
 	ExitCode                int       `json:"exit_code" gorm:"uniqueIndex:compositeindex"`
 	StartTime               time.Time `json:"start_time" gorm:"uniqueIndex:compositeindex"`
 	EndTime                 time.Time `json:"end_time" gorm:"uniqueIndex:compositeindex"`
+	DeviceId                string    `json:"device_id" gorm:"uniqueIndex:compositeindex"`
 }
 
 func sha256hmac(key, additionalData string) []byte {

client/lib/lib.go

@@ -137,6 +137,13 @@ func BuildHistoryEntry(args []string) (*data.HistoryEntry, error) {
 	}
 	entry.Hostname = hostname
 
+	// device ID
+	config, err := GetConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get device ID when building history entry: %v", err)
+	}
+	entry.DeviceId = config.DeviceId
+
 	return &entry, nil
 }
 
@@ -686,15 +693,15 @@ func ApiGet(path string) ([]byte, error) {
 	start := time.Now()
 	resp, err := http.Get(getServerHostname() + path)
 	if err != nil {
-		return nil, fmt.Errorf("failed to GET %s: %v", path, err)
+		return nil, fmt.Errorf("failed to GET %s%s: %v", getServerHostname(), path, err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("failed to GET %s: status_code=%d", path, resp.StatusCode)
+		return nil, fmt.Errorf("failed to GET %s%s: status_code=%d", getServerHostname(), path, resp.StatusCode)
 	}
 	respBody, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read response body from GET %s: %v", path, err)
+		return nil, fmt.Errorf("failed to read response body from GET %s%s: %v", getServerHostname(), path, err)
 	}
 	duration := time.Since(start)
 	GetLogger().Printf("ApiGet(%#v): %s\n", path, duration.String())
@@ -796,3 +803,7 @@ func setCodesigningXattrs(downloadInfo shared.UpdateInfo, filename string) error
 	setXattr(filename, string(xattrDump))
 	return nil
 }
+
+func IsOfflineError(err error) bool {
+	return strings.Contains(err.Error(), "dial tcp: lookup api.hishtory.dev") || strings.Contains(err.Error(), "read: connection reset by peer")
+}

hishtory.go

@@ -46,6 +46,7 @@ func main() {
 		if len(os.Args) == 3 && os.Args[2] == "-v" {
 			fmt.Printf("User ID: %s\n", data.UserId(config.UserSecret))
 			fmt.Printf("Device ID: %s\n", config.DeviceId)
+			printDumpStatus(config)
 		}
 		fmt.Printf("Commit Hash: %s\n", GitCommit)
 	case "update":
@@ -55,6 +56,17 @@ func main() {
 	}
 }
 
+func printDumpStatus(config lib.ClientConfig) {
+	respBody, err := lib.ApiGet("/api/v1/get-dump-requests?user_id=" + data.UserId(config.UserSecret))
+	if err != nil {
+		lib.CheckFatalError(err)
+	}
+	var dumpRequests []*shared.DumpRequest
+	err = json.Unmarshal(respBody, &dumpRequests)
+	lib.CheckFatalError(err)
+	fmt.Printf("Dump Requests: %#v\n", dumpRequests)
+}
+
 func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error {
 	config, err := lib.GetConfig()
 	if err != nil {
@@ -69,11 +81,13 @@ func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error {
 	if err != nil {
 		return fmt.Errorf("failed to load JSON response: %v", err)
 	}
+	// fmt.Printf("this device id=%s, user id=%s\n", config.DeviceId, data.UserId(config.UserSecret))
 	for _, entry := range retrievedEntries {
 		decEntry, err := data.DecryptHistoryEntry(config.UserSecret, *entry)
 		if err != nil {
 			return fmt.Errorf("failed to decrypt history entry from server: %v", err)
 		}
+		// fmt.Printf("received entry: %#v\n", decEntry)
 		lib.AddToDbIfNew(db, decEntry)
 	}
 	return nil
@@ -82,7 +96,14 @@ func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error {
 func query(query string) {
 	db, err := lib.OpenLocalSqliteDb()
 	lib.CheckFatalError(err)
-	lib.CheckFatalError(retrieveAdditionalEntriesFromRemote(db))
+	err = retrieveAdditionalEntriesFromRemote(db)
+	if err != nil {
+		if lib.IsOfflineError(err) {
+			fmt.Println("Warning: hishtory is offline so this may be missing recent results from your other machines!")
+		} else {
+			lib.CheckFatalError(err)
+		}
+	}
 	lib.CheckFatalError(displayBannerIfSet())
 	data, err := data.Search(db, query, 25)
 	lib.CheckFatalError(err)
@@ -133,7 +154,7 @@ func saveHistoryEntry() {
 	lib.CheckFatalError(err)
 	_, err = lib.ApiPost("/api/v1/submit", "application/json", jsonValue)
 	if err != nil {
-		if strings.Contains(err.Error(), "dial tcp: lookup api.hishtory.dev") || strings.Contains(err.Error(), "read: connection reset by peer") {
+		if lib.IsOfflineError(err) {
 			// TODO: Somehow handle this and don't completely lose it
 			lib.GetLogger().Printf("Failed to remotely persist hishtory entry because the device is offline!")
 		} else {
@@ -143,7 +164,14 @@ func saveHistoryEntry() {
 
 	// Check if there is a pending dump request and reply to it if so
 	resp, err := lib.ApiGet("/api/v1/get-dump-requests?user_id=" + data.UserId(config.UserSecret) + "&device_id=" + config.DeviceId)
-	lib.CheckFatalError(err)
+	if err != nil {
+		if lib.IsOfflineError(err) {
+			// It is fine to just ignore this, the next command will retry the API and eventually we will respond to any pending dump requests
+			resp = []byte("[]")
+		} else {
+			lib.CheckFatalError(err)
+		}
+	}
 	var dumpRequests []*shared.DumpRequest
 	err = json.Unmarshal(resp, &dumpRequests)
 	lib.CheckFatalError(err)
@@ -160,7 +188,7 @@ func saveHistoryEntry() {
 		reqBody, err := json.Marshal(encEntries)
 		lib.CheckFatalError(err)
 		for _, dumpRequest := range dumpRequests {
-			_, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId, "application/json", reqBody)
+			_, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId+"&source_device_id="+config.DeviceId, "application/json", reqBody)
 			lib.CheckFatalError(err)
 		}
 	}
@@ -169,7 +197,14 @@ func saveHistoryEntry() {
 func export() {
 	db, err := lib.OpenLocalSqliteDb()
 	lib.CheckFatalError(err)
-	lib.CheckFatalError(retrieveAdditionalEntriesFromRemote(db))
+	err = retrieveAdditionalEntriesFromRemote(db)
+	if err != nil {
+		if lib.IsOfflineError(err) {
+			fmt.Println("Warning: hishtory is offline so this may be missing recent results from your other machines!")
+		} else {
+			lib.CheckFatalError(err)
+		}
+	}
 	data, err := data.Search(db, "", 0)
 	lib.CheckFatalError(err)
 	for i := len(data) - 1; i >= 0; i-- {

shared/testutils.go

@@ -13,6 +13,11 @@ import (
 	"time"
 )
 
+const (
+	DB_WAL_PATH = DB_PATH + "-wal"
+	DB_SHM_PATH = DB_PATH + "-shm"
+)
+
 func ResetLocalState(t *testing.T) {
 	homedir, err := os.UserHomeDir()
 	if err != nil {
@@ -20,6 +25,7 @@ func ResetLocalState(t *testing.T) {
 	}
 
 	_ = os.Remove(path.Join(homedir, HISHTORY_PATH, DB_PATH))
+	_ = os.Remove(path.Join(homedir, HISHTORY_PATH, DB_WAL_PATH))
 	_ = os.Remove(path.Join(homedir, HISHTORY_PATH, CONFIG_PATH))
 	_ = os.Remove(path.Join(homedir, HISHTORY_PATH, "hishtory"))
 	_ = os.Remove(path.Join(homedir, HISHTORY_PATH, "config.sh"))
@@ -27,22 +33,30 @@ func ResetLocalState(t *testing.T) {
 }
 
 func BackupAndRestore(t *testing.T) func() {
+	return BackupAndRestoreWithId(t, "")
+}
+
+func BackupAndRestoreWithId(t *testing.T, id string) func() {
 	homedir, err := os.UserHomeDir()
 	if err != nil {
 		t.Fatalf("failed to retrieve homedir: %v", err)
 	}
 
-	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_PATH), path.Join(homedir, HISHTORY_PATH, DB_PATH+".bak"))
-	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, CONFIG_PATH), path.Join(homedir, HISHTORY_PATH, CONFIG_PATH+".bak"))
-	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "hishtory"), path.Join(homedir, HISHTORY_PATH, "hishtory.bak"))
-	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.sh"), path.Join(homedir, HISHTORY_PATH, "config.sh.bak"))
-	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.zsh"), path.Join(homedir, HISHTORY_PATH, "config.zsh.bak"))
+	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_PATH), path.Join(homedir, HISHTORY_PATH, DB_PATH+id+".bak"))
+	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_WAL_PATH), path.Join(homedir, HISHTORY_PATH, DB_WAL_PATH+id+".bak"))
+	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_SHM_PATH), path.Join(homedir, HISHTORY_PATH, DB_SHM_PATH+id+".bak"))
+	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, CONFIG_PATH), path.Join(homedir, HISHTORY_PATH, CONFIG_PATH+id+".bak"))
+	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "hishtory"), path.Join(homedir, HISHTORY_PATH, "hishtory"+id+".bak"))
+	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.sh"), path.Join(homedir, HISHTORY_PATH, "config.sh"+id+"bak"))
+	_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.zsh"), path.Join(homedir, HISHTORY_PATH, "config.zsh"+id+".bak"))
 	return func() {
-		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_PATH+".bak"), path.Join(homedir, HISHTORY_PATH, DB_PATH))
-		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, CONFIG_PATH+".bak"), path.Join(homedir, HISHTORY_PATH, CONFIG_PATH))
-		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "hishtory.bak"), path.Join(homedir, HISHTORY_PATH, "hishtory"))
-		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.sh.bak"), path.Join(homedir, HISHTORY_PATH, "config.sh"))
-		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.zsh.bak"), path.Join(homedir, HISHTORY_PATH, "config.zsh"))
+		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_PATH+id+".bak"), path.Join(homedir, HISHTORY_PATH, DB_PATH))
+		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_WAL_PATH+id+".bak"), path.Join(homedir, HISHTORY_PATH, DB_WAL_PATH))
+		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, DB_SHM_PATH+id+".bak"), path.Join(homedir, HISHTORY_PATH, DB_SHM_PATH))
+		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, CONFIG_PATH+id+".bak"), path.Join(homedir, HISHTORY_PATH, CONFIG_PATH))
+		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "hishtory"+id+".bak"), path.Join(homedir, HISHTORY_PATH, "hishtory"))
+		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.sh"+id+".bak"), path.Join(homedir, HISHTORY_PATH, "config.sh"))
+		_ = os.Rename(path.Join(homedir, HISHTORY_PATH, "config.zsh"+id+".bak"), path.Join(homedir, HISHTORY_PATH, "config.zsh"))
 	}
 }