Offline first! Now if a devide is offline it will detect this and upload the entries later

client/client_test.go

@@ -126,6 +126,7 @@ func TestParameterized(t *testing.T) {
 		t.Run("testExportWithQuery/"+tester.ShellName(), func(t *testing.T) { testExportWithQuery(t, tester) })
 		t.Run("testHelpCommand/"+tester.ShellName(), func(t *testing.T) { testHelpCommand(t, tester) })
 		t.Run("testStripBashTimePrefix/"+tester.ShellName(), func(t *testing.T) { testStripBashTimePrefix(t, tester) })
+		t.Run("testReuploadHistoryEntries/"+tester.ShellName(), func(t *testing.T) { testReuploadHistoryEntries(t, tester) })
 	}
 }
 
@@ -1203,4 +1204,51 @@ func testStripBashTimePrefix(t *testing.T, tester shellTester) {
 	}
 }
 
+func testReuploadHistoryEntries(t *testing.T, tester shellTester) {
+	// Setup
+	defer shared.BackupAndRestore(t)()
+
+	// Init an initial device
+	userSecret := installHishtory(t, tester, "")
+
+	// Set up a second device
+	restoreFirstProfile := shared.BackupAndRestoreWithId(t, "-install1")
+	installHishtory(t, tester, userSecret)
+
+	// Device 2: Record a command
+	tester.RunInteractiveShell(t, `echo 1`)
+
+	// Device 2: Record a command with a simulated network error
+	tester.RunInteractiveShell(t, `echo 2; export HISHTORY_SIMULATE_NETWORK_ERROR=1; echo 3`)
+
+	// Device 1: Run an export and confirm that the network only contains the first command
+	restoreSecondProfile := shared.BackupAndRestoreWithId(t, "-install2")
+	restoreFirstProfile()
+	out := tester.RunInteractiveShell(t, "hishtory export | grep -v pipefail")
+	expectedOutput := "echo 1\n"
+	if diff := cmp.Diff(expectedOutput, out); diff != "" {
+		t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out)
+	}
+
+	// Device 2: Run another command but with the network re-enabled
+	restoreFirstProfile = shared.BackupAndRestoreWithId(t, "-install1")
+	restoreSecondProfile()
+	tester.RunInteractiveShell(t, `unset HISHTORY_SIMULATE_NETWORK_ERROR; echo 4`)
+
+	// Device 2: Run export which contains all results (as it did all along since it is stored offline)
+	out = tester.RunInteractiveShell(t, "hishtory export | grep -v pipefail")
+	expectedOutput = "echo 1\necho 2; export HISHTORY_SIMULATE_NETWORK_ERROR=1; echo 3\nunset HISHTORY_SIMULATE_NETWORK_ERROR; echo 4\n"
+	if diff := cmp.Diff(expectedOutput, out); diff != "" {
+		t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out)
+	}
+
+	// Device 1: Now it too sees all the results
+	restoreFirstProfile()
+	out = tester.RunInteractiveShell(t, "hishtory export | grep -v pipefail")
+	expectedOutput = "echo 1\necho 2; export HISHTORY_SIMULATE_NETWORK_ERROR=1; echo 3\nunset HISHTORY_SIMULATE_NETWORK_ERROR; echo 4\n"
+	if diff := cmp.Diff(expectedOutput, out); diff != "" {
+		t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out)
+	}
+}
+
 // TODO: write a test that runs hishtroy export | grep -v pipefail and then see if that shows up in query/export, I think there is weird behavior here

client/data/data.go

@@ -206,6 +206,7 @@ func parseAtomizedToken(token string) (string, interface{}, error) {
 		}
 		return "(CAST(strftime(\"%s\",start_time) AS INTEGER) < ?)", t.Unix(), nil
 	case "after":
+		// TODO: This doesn't support precise timestamps containg a space. Can we do better here?
 		t, err := parseTimeGenerously(val)
 		if err != nil {
 			return "", nil, fmt.Errorf("failed to parse after:%s as a timestamp: %v", val, err)

client/lib/lib.go

@@ -339,10 +339,14 @@ func DisplayResults(results []*data.HistoryEntry) {
 }
 
 type ClientConfig struct {
-	UserSecret           string `json:"user_secret"`
-	IsEnabled            bool   `json:"is_enabled"`
-	DeviceId             string `json:"device_id"`
+	UserSecret string `json:"user_secret"`
+	IsEnabled  bool   `json:"is_enabled"`
+	DeviceId   string `json:"device_id"`
+	// Used for skipping history entries prefixed with a space in bash
 	LastSavedHistoryLine string `json:"last_saved_history_line"`
+	// Used for uploading history entries that we failed to upload due to a missing network connection
+	HaveMissedUploads     bool  `json:"have_missed_uploads"`
+	MissedUploadTimestamp int64 `json:"missed_upload_timestamp"`
 }
 
 func GetConfig() (ClientConfig, error) {
@@ -865,6 +869,9 @@ func OpenLocalSqliteDb() (*gorm.DB, error) {
 }
 
 func ApiGet(path string) ([]byte, error) {
+	if os.Getenv("HISHTORY_SIMULATE_NETWORK_ERROR") != "" {
+		return nil, fmt.Errorf("simulated network error: dial tcp: lookup api.hishtory.dev")
+	}
 	start := time.Now()
 	resp, err := http.Get(getServerHostname() + path)
 	if err != nil {
@@ -884,6 +891,9 @@ func ApiGet(path string) ([]byte, error) {
 }
 
 func ApiPost(path, contentType string, data []byte) ([]byte, error) {
+	if os.Getenv("HISHTORY_SIMULATE_NETWORK_ERROR") != "" {
+		return nil, fmt.Errorf("simulated network error: dial tcp: lookup api.hishtory.dev")
+	}
 	start := time.Now()
 	resp, err := http.Post(getServerHostname()+path, contentType, bytes.NewBuffer(data))
 	if err != nil {
@@ -933,3 +943,16 @@ func ReliableDbCreate(db *gorm.DB, entry interface{}) error {
 	}
 	return fmt.Errorf("failed to create DB entry even with %d retries: %v", i, err)
 }
+
+func EncryptAndMarshal(config ClientConfig, entry *data.HistoryEntry) ([]byte, error) {
+	encEntry, err := data.EncryptHistoryEntry(config.UserSecret, *entry)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encrypt history entry")
+	}
+	encEntry.DeviceId = config.DeviceId
+	jsonValue, err := json.Marshal([]shared.EncHistoryEntry{encEntry})
+	if err != nil {
+		return jsonValue, fmt.Errorf("failed to marshal encrypted history entry: %v", err)
+	}
+	return jsonValue, nil
+}

hishtory.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"strings"
+	"time"
 
 	"gorm.io/gorm"
 
@@ -23,6 +24,7 @@ func main() {
 	}
 	switch os.Args[1] {
 	case "saveHistoryEntry":
+		lib.CheckFatalError(maybeUploadSkippedHistoryEntries())
 		saveHistoryEntry()
 	case "query":
 		query(strings.Join(os.Args[2:], " "))
@@ -158,6 +160,48 @@ func displayBannerIfSet() error {
 	return nil
 }
 
+func maybeUploadSkippedHistoryEntries() error {
+	config, err := lib.GetConfig()
+	if err != nil {
+		return err
+	}
+	if !config.HaveMissedUploads {
+		return nil
+	}
+
+	// Upload the missing entries
+	db, err := lib.OpenLocalSqliteDb()
+	if err != nil {
+		return nil
+	}
+	query := fmt.Sprintf("after:%s", time.Unix(config.MissedUploadTimestamp, 0).Format("2006-01-02"))
+	entries, err := data.Search(db, query, 0)
+	if err != nil {
+		return fmt.Errorf("failed to retrieve history entries that haven't been uploaded yet: %v", err)
+	}
+	lib.GetLogger().Printf("Uploading %d history entries that previously failed to upload (query=%#v)\n", len(entries), query)
+	for _, entry := range entries {
+		jsonValue, err := lib.EncryptAndMarshal(config, entry)
+		if err != nil {
+			return err
+		}
+		_, err = lib.ApiPost("/api/v1/submit", "application/json", jsonValue)
+		if err != nil {
+			// Failed to upload the history entry, so we must still be offline. So just return nil and we'll try again later.
+			return nil
+		}
+	}
+
+	// Mark down that we persisted it
+	config.HaveMissedUploads = false
+	config.MissedUploadTimestamp = 0
+	err = lib.SetConfig(config)
+	if err != nil {
+		return fmt.Errorf("failed to mark a history entry as uploaded: %v", err)
+	}
+	return nil
+}
+
 func saveHistoryEntry() {
 	config, err := lib.GetConfig()
 	if err != nil {
@@ -181,16 +225,17 @@ func saveHistoryEntry() {
 	lib.CheckFatalError(err)
 
 	// Persist it remotely
-	encEntry, err := data.EncryptHistoryEntry(config.UserSecret, *entry)
-	lib.CheckFatalError(err)
-	encEntry.DeviceId = config.DeviceId
-	jsonValue, err := json.Marshal([]shared.EncHistoryEntry{encEntry})
+	jsonValue, err := lib.EncryptAndMarshal(config, entry)
 	lib.CheckFatalError(err)
 	_, err = lib.ApiPost("/api/v1/submit", "application/json", jsonValue)
 	if err != nil {
 		if lib.IsOfflineError(err) {
-			// TODO: Somehow handle this and don't completely lose it
 			lib.GetLogger().Printf("Failed to remotely persist hishtory entry because the device is offline!")
+			if !config.HaveMissedUploads {
+				config.HaveMissedUploads = true
+				config.MissedUploadTimestamp = time.Now().Unix()
+				lib.CheckFatalError(lib.SetConfig(config))
+			}
 		} else {
 			lib.CheckFatalError(err)
 		}