diff --git a/client/client_test.go b/client/client_test.go index fed261e..3c2b15a 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -126,6 +126,7 @@ func TestParameterized(t *testing.T) { t.Run("testExportWithQuery/"+tester.ShellName(), func(t *testing.T) { testExportWithQuery(t, tester) }) t.Run("testHelpCommand/"+tester.ShellName(), func(t *testing.T) { testHelpCommand(t, tester) }) t.Run("testStripBashTimePrefix/"+tester.ShellName(), func(t *testing.T) { testStripBashTimePrefix(t, tester) }) + t.Run("testReuploadHistoryEntries/"+tester.ShellName(), func(t *testing.T) { testReuploadHistoryEntries(t, tester) }) } } @@ -1203,4 +1204,51 @@ func testStripBashTimePrefix(t *testing.T, tester shellTester) { } } +func testReuploadHistoryEntries(t *testing.T, tester shellTester) { + // Setup + defer shared.BackupAndRestore(t)() + + // Init an initial device + userSecret := installHishtory(t, tester, "") + + // Set up a second device + restoreFirstProfile := shared.BackupAndRestoreWithId(t, "-install1") + installHishtory(t, tester, userSecret) + + // Device 2: Record a command + tester.RunInteractiveShell(t, `echo 1`) + + // Device 2: Record a command with a simulated network error + tester.RunInteractiveShell(t, `echo 2; export HISHTORY_SIMULATE_NETWORK_ERROR=1; echo 3`) + + // Device 1: Run an export and confirm that the network only contains the first command + restoreSecondProfile := shared.BackupAndRestoreWithId(t, "-install2") + restoreFirstProfile() + out := tester.RunInteractiveShell(t, "hishtory export | grep -v pipefail") + expectedOutput := "echo 1\n" + if diff := cmp.Diff(expectedOutput, out); diff != "" { + t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out) + } + + // Device 2: Run another command but with the network re-enabled + restoreFirstProfile = shared.BackupAndRestoreWithId(t, "-install1") + restoreSecondProfile() + tester.RunInteractiveShell(t, `unset HISHTORY_SIMULATE_NETWORK_ERROR; echo 4`) + + // Device 2: Run export which contains all results (as it did all along since it is stored offline) + out = tester.RunInteractiveShell(t, "hishtory export | grep -v pipefail") + expectedOutput = "echo 1\necho 2; export HISHTORY_SIMULATE_NETWORK_ERROR=1; echo 3\nunset HISHTORY_SIMULATE_NETWORK_ERROR; echo 4\n" + if diff := cmp.Diff(expectedOutput, out); diff != "" { + t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out) + } + + // Device 1: Now it too sees all the results + restoreFirstProfile() + out = tester.RunInteractiveShell(t, "hishtory export | grep -v pipefail") + expectedOutput = "echo 1\necho 2; export HISHTORY_SIMULATE_NETWORK_ERROR=1; echo 3\nunset HISHTORY_SIMULATE_NETWORK_ERROR; echo 4\n" + if diff := cmp.Diff(expectedOutput, out); diff != "" { + t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out) + } +} + // TODO: write a test that runs hishtroy export | grep -v pipefail and then see if that shows up in query/export, I think there is weird behavior here diff --git a/client/data/data.go b/client/data/data.go index 1dc431a..959f2b1 100644 --- a/client/data/data.go +++ b/client/data/data.go @@ -206,6 +206,7 @@ func parseAtomizedToken(token string) (string, interface{}, error) { } return "(CAST(strftime(\"%s\",start_time) AS INTEGER) < ?)", t.Unix(), nil case "after": + // TODO: This doesn't support precise timestamps containg a space. Can we do better here? t, err := parseTimeGenerously(val) if err != nil { return "", nil, fmt.Errorf("failed to parse after:%s as a timestamp: %v", val, err) diff --git a/client/lib/lib.go b/client/lib/lib.go index b3f4789..44a5d9d 100644 --- a/client/lib/lib.go +++ b/client/lib/lib.go @@ -339,10 +339,14 @@ func DisplayResults(results []*data.HistoryEntry) { } type ClientConfig struct { - UserSecret string `json:"user_secret"` - IsEnabled bool `json:"is_enabled"` - DeviceId string `json:"device_id"` + UserSecret string `json:"user_secret"` + IsEnabled bool `json:"is_enabled"` + DeviceId string `json:"device_id"` + // Used for skipping history entries prefixed with a space in bash LastSavedHistoryLine string `json:"last_saved_history_line"` + // Used for uploading history entries that we failed to upload due to a missing network connection + HaveMissedUploads bool `json:"have_missed_uploads"` + MissedUploadTimestamp int64 `json:"missed_upload_timestamp"` } func GetConfig() (ClientConfig, error) { @@ -865,6 +869,9 @@ func OpenLocalSqliteDb() (*gorm.DB, error) { } func ApiGet(path string) ([]byte, error) { + if os.Getenv("HISHTORY_SIMULATE_NETWORK_ERROR") != "" { + return nil, fmt.Errorf("simulated network error: dial tcp: lookup api.hishtory.dev") + } start := time.Now() resp, err := http.Get(getServerHostname() + path) if err != nil { @@ -884,6 +891,9 @@ func ApiGet(path string) ([]byte, error) { } func ApiPost(path, contentType string, data []byte) ([]byte, error) { + if os.Getenv("HISHTORY_SIMULATE_NETWORK_ERROR") != "" { + return nil, fmt.Errorf("simulated network error: dial tcp: lookup api.hishtory.dev") + } start := time.Now() resp, err := http.Post(getServerHostname()+path, contentType, bytes.NewBuffer(data)) if err != nil { @@ -933,3 +943,16 @@ func ReliableDbCreate(db *gorm.DB, entry interface{}) error { } return fmt.Errorf("failed to create DB entry even with %d retries: %v", i, err) } + +func EncryptAndMarshal(config ClientConfig, entry *data.HistoryEntry) ([]byte, error) { + encEntry, err := data.EncryptHistoryEntry(config.UserSecret, *entry) + if err != nil { + return nil, fmt.Errorf("failed to encrypt history entry") + } + encEntry.DeviceId = config.DeviceId + jsonValue, err := json.Marshal([]shared.EncHistoryEntry{encEntry}) + if err != nil { + return jsonValue, fmt.Errorf("failed to marshal encrypted history entry: %v", err) + } + return jsonValue, nil +} diff --git a/hishtory.go b/hishtory.go index 04ed062..400e083 100644 --- a/hishtory.go +++ b/hishtory.go @@ -6,6 +6,7 @@ import ( "log" "os" "strings" + "time" "gorm.io/gorm" @@ -23,6 +24,7 @@ func main() { } switch os.Args[1] { case "saveHistoryEntry": + lib.CheckFatalError(maybeUploadSkippedHistoryEntries()) saveHistoryEntry() case "query": query(strings.Join(os.Args[2:], " ")) @@ -158,6 +160,48 @@ func displayBannerIfSet() error { return nil } +func maybeUploadSkippedHistoryEntries() error { + config, err := lib.GetConfig() + if err != nil { + return err + } + if !config.HaveMissedUploads { + return nil + } + + // Upload the missing entries + db, err := lib.OpenLocalSqliteDb() + if err != nil { + return nil + } + query := fmt.Sprintf("after:%s", time.Unix(config.MissedUploadTimestamp, 0).Format("2006-01-02")) + entries, err := data.Search(db, query, 0) + if err != nil { + return fmt.Errorf("failed to retrieve history entries that haven't been uploaded yet: %v", err) + } + lib.GetLogger().Printf("Uploading %d history entries that previously failed to upload (query=%#v)\n", len(entries), query) + for _, entry := range entries { + jsonValue, err := lib.EncryptAndMarshal(config, entry) + if err != nil { + return err + } + _, err = lib.ApiPost("/api/v1/submit", "application/json", jsonValue) + if err != nil { + // Failed to upload the history entry, so we must still be offline. So just return nil and we'll try again later. + return nil + } + } + + // Mark down that we persisted it + config.HaveMissedUploads = false + config.MissedUploadTimestamp = 0 + err = lib.SetConfig(config) + if err != nil { + return fmt.Errorf("failed to mark a history entry as uploaded: %v", err) + } + return nil +} + func saveHistoryEntry() { config, err := lib.GetConfig() if err != nil { @@ -181,16 +225,17 @@ func saveHistoryEntry() { lib.CheckFatalError(err) // Persist it remotely - encEntry, err := data.EncryptHistoryEntry(config.UserSecret, *entry) - lib.CheckFatalError(err) - encEntry.DeviceId = config.DeviceId - jsonValue, err := json.Marshal([]shared.EncHistoryEntry{encEntry}) + jsonValue, err := lib.EncryptAndMarshal(config, entry) lib.CheckFatalError(err) _, err = lib.ApiPost("/api/v1/submit", "application/json", jsonValue) if err != nil { if lib.IsOfflineError(err) { - // TODO: Somehow handle this and don't completely lose it lib.GetLogger().Printf("Failed to remotely persist hishtory entry because the device is offline!") + if !config.HaveMissedUploads { + config.HaveMissedUploads = true + config.MissedUploadTimestamp = time.Now().Unix() + lib.CheckFatalError(lib.SetConfig(config)) + } } else { lib.CheckFatalError(err) }