Merge pull request #111 from ddworken/sync-bug

Fix bug in offline syncing resumption

backend/server/internal/database/db.go

@@ -63,6 +63,26 @@ func (db *DB) AddDatabaseTables() error {
 	return nil
 }
 
+func (db *DB) CreateIndices() error {
+	// Note: If adding a new index here, consider manually running it on the prod DB using CONCURRENTLY to
+	// make server startup non-blocking. The benefit of this function is primarily for other people so they
+	// don't have to manually create these indexes.
+	var indices = []string{
+		`CREATE INDEX IF NOT EXISTS entry_id_idx ON enc_history_entries USING btree(encrypted_id);`,
+		`CREATE INDEX IF NOT EXISTS device_id_idx ON enc_history_entries USING btree(device_id);`,
+		`CREATE INDEX IF NOT EXISTS read_count_idx ON enc_history_entries USING btree(read_count);`,
+		`CREATE INDEX IF NOT EXISTS redact_idx ON enc_history_entries USING btree(user_id, device_id, date);`,
+		`CREATE INDEX IF NOT EXISTS del_user_idx ON deletion_requests USING btree(user_id);`,
+	}
+	for _, index := range indices {
+		r := db.Exec(index)
+		if r.Error != nil {
+			return fmt.Errorf("failed to execute index creation sql=%#v: %w", index, r.Error)
+		}
+	}
+	return nil
+}
+
 func (db *DB) Close() error {
 	rawDB, err := db.DB.DB()
 	if err != nil {
@@ -152,7 +172,8 @@ func (db *DB) DumpRequestDeleteForUserAndDevice(ctx context.Context, userID, dev
 func (db *DB) ApplyDeletionRequestsToBackend(ctx context.Context, request *shared.DeletionRequest) (int64, error) {
 	tx := db.WithContext(ctx).Where("false")
 	for _, message := range request.Messages.Ids {
-		tx = tx.Or(db.WithContext(ctx).Where("user_id = ? AND device_id = ? AND date = ?", request.UserId, message.DeviceId, message.Date))
+		// Note that we do an OR with date or the ID matching since the ID is not always recorded for older history entries.
+		tx = tx.Or(db.WithContext(ctx).Where("user_id = ? AND device_id = ? AND (date = ? OR encrypted_id = ?)", request.UserId, message.DeviceId, message.EndTime, message.EntryId))
 	}
 	result := tx.Delete(&shared.EncHistoryEntry{})
 	if tx.Error != nil {

backend/server/internal/server/api_handlers.go

@@ -1,7 +1,6 @@
 package server
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"html"
@@ -49,34 +48,24 @@ func (s *Server) apiSubmitHandler(w http.ResponseWriter, r *http.Request) {
 		s.statsd.Count("hishtory.submit", int64(len(devices)), []string{}, 1.0)
 	}
 
+	resp := shared.SubmitResponse{}
+
 	deviceId := getOptionalQueryParam(r, "source_device_id", s.isTestEnvironment)
-	resp := shared.SubmitResponse{
+	if deviceId != "" {
-		HaveDumpRequests:     s.haveDumpRequests(r.Context(), userId, deviceId),
+		dumpRequests, err := s.db.DumpRequestForUserAndDevice(r.Context(), userId, deviceId)
-		HaveDeletionRequests: s.haveDeletionRequests(r.Context(), userId, deviceId),
+		checkGormError(err)
+		resp.DumpRequests = dumpRequests
+
+		deletionRequests, err := s.db.DeletionRequestsForUserAndDevice(r.Context(), userId, deviceId)
+		checkGormError(err)
+		resp.DeletionRequests = deletionRequests
 	}
+
 	if err := json.NewEncoder(w).Encode(resp); err != nil {
 		panic(err)
 	}
 }
 
-func (s *Server) haveDumpRequests(ctx context.Context, userId, deviceId string) bool {
-	if userId == "" || deviceId == "" {
-		return true
-	}
-	dumpRequests, err := s.db.DumpRequestForUserAndDevice(ctx, userId, deviceId)
-	checkGormError(err)
-	return len(dumpRequests) > 0
-}
-
-func (s *Server) haveDeletionRequests(ctx context.Context, userId, deviceId string) bool {
-	if userId == "" || deviceId == "" {
-		return true
-	}
-	deletionRequests, err := s.db.DeletionRequestsForUserAndDevice(ctx, userId, deviceId)
-	checkGormError(err)
-	return len(deletionRequests) > 0
-}
-
 func (s *Server) apiBootstrapHandler(w http.ResponseWriter, r *http.Request) {
 	userId := getRequiredQueryParam(r, "user_id")
 	deviceId := getRequiredQueryParam(r, "device_id")

backend/server/internal/server/server_test.go

@@ -81,7 +81,8 @@ func TestESubmitThenQuery(t *testing.T) {
 	w := httptest.NewRecorder()
 	s.apiSubmitHandler(w, submitReq)
 	require.Equal(t, 200, w.Result().StatusCode)
-	require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: false}, deserializeSubmitResponse(t, w))
+	require.Empty(t, deserializeSubmitResponse(t, w).DeletionRequests)
+	require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
 
 	// Query for device id 1
 	w = httptest.NewRecorder()
@@ -346,7 +347,8 @@ func TestDeletionRequests(t *testing.T) {
 	w := httptest.NewRecorder()
 	s.apiSubmitHandler(w, submitReq)
 	require.Equal(t, 200, w.Result().StatusCode)
-	require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: false}, deserializeSubmitResponse(t, w))
+	require.Empty(t, deserializeSubmitResponse(t, w).DeletionRequests)
+	require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
 
 	// And another entry for user1
 	entry2 := testutils.MakeFakeHistoryEntry("ls /foo/bar")
@@ -359,7 +361,8 @@ func TestDeletionRequests(t *testing.T) {
 	w = httptest.NewRecorder()
 	s.apiSubmitHandler(w, submitReq)
 	require.Equal(t, 200, w.Result().StatusCode)
-	require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: false}, deserializeSubmitResponse(t, w))
+	require.Empty(t, deserializeSubmitResponse(t, w).DeletionRequests)
+	require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
 
 	// And an entry for user2 that has the same timestamp as the previous entry
 	entry3 := testutils.MakeFakeHistoryEntry("ls /foo/bar")
@@ -373,7 +376,8 @@ func TestDeletionRequests(t *testing.T) {
 	w = httptest.NewRecorder()
 	s.apiSubmitHandler(w, submitReq)
 	require.Equal(t, 200, w.Result().StatusCode)
-	require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: false}, deserializeSubmitResponse(t, w))
+	require.Empty(t, deserializeSubmitResponse(t, w).DeletionRequests)
+	require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
 
 	// Query for device id 1
 	w = httptest.NewRecorder()
@@ -411,7 +415,7 @@ func TestDeletionRequests(t *testing.T) {
 		UserId:   data.UserId("dkey"),
 		SendTime: delReqTime,
 		Messages: shared.MessageIdentifiers{Ids: []shared.MessageIdentifier{
-			{DeviceId: devId1, Date: entry1.EndTime},
+			{DeviceId: devId1, EndTime: entry1.EndTime},
 		}},
 	}
 	reqBody, err = json.Marshal(delReq)
@@ -485,7 +489,8 @@ func TestDeletionRequests(t *testing.T) {
 	w = httptest.NewRecorder()
 	s.apiSubmitHandler(w, submitReq)
 	require.Equal(t, 200, w.Result().StatusCode)
-	require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: true}, deserializeSubmitResponse(t, w))
+	require.NotEmpty(t, deserializeSubmitResponse(t, w).DeletionRequests)
+	require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
 
 	// Query for deletion requests
 	w = httptest.NewRecorder()
@@ -507,7 +512,7 @@ func TestDeletionRequests(t *testing.T) {
 		SendTime:            delReqTime,
 		ReadCount:           1,
 		Messages: shared.MessageIdentifiers{Ids: []shared.MessageIdentifier{
-			{DeviceId: devId1, Date: entry1.EndTime},
+			{DeviceId: devId1, EndTime: entry1.EndTime},
 		}},
 	}
 	if diff := deep.Equal(*deletionRequest, expected); diff != nil {
@@ -585,7 +590,8 @@ func TestCleanDatabaseNoErrors(t *testing.T) {
 	w := httptest.NewRecorder()
 	s.apiSubmitHandler(w, submitReq)
 	require.Equal(t, 200, w.Result().StatusCode)
-	require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: false}, deserializeSubmitResponse(t, w))
+	require.Empty(t, deserializeSubmitResponse(t, w).DeletionRequests)
+	require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
 
 	// Call cleanDatabase and just check that there are no panics
 	testutils.Check(t, DB.Clean(context.TODO()))

backend/server/server.go

@@ -95,6 +95,10 @@ func OpenDB() (*database.DB, error) {
 			return nil, fmt.Errorf("failed to create underlying DB tables: %w", err)
 		}
 	}
+	err := db.CreateIndices()
+	if err != nil {
+		return nil, err
+	}
 	return db, nil
 }
 

client/client_test.go

@@ -2364,15 +2364,24 @@ func testPresaving(t *testing.T, tester shellTester) {
 	out = tester.RunInteractiveShell(t, ` hishtory query sleep 13371337 -export -tquery`)
 	testutils.CompareGoldens(t, out, "testPresaving-query")
 
-	// And the same for tquery
+	// Create a new device, and confirm it shows up there too
-	// out = captureTerminalOutputWithComplexCommands(t, tester,
+	restoreDevice1 := testutils.BackupAndRestoreWithId(t, "device1")
-	// 	[]TmuxCommand{
+	installHishtory(t, tester, userSecret)
-	// 		{Keys: "hishtory SPACE tquery ENTER", ExtraDelay: 2.0},
+	tester.RunInteractiveShell(t, ` hishtory config-set displayed-columns CWD Runtime Command`)
-	// 		{Keys: "sleep SPACE 13371337 SPACE -export SPACE -tquery"}})
+	out = tester.RunInteractiveShell(t, ` hishtory query sleep 13371337 -export -tquery`)
-	// out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1])
+	testutils.CompareGoldens(t, out, "testPresaving-query")
-	// testutils.CompareGoldens(t, out, "testPresaving-tquery")
+
-	//
+	// And then redact it from device2
-	// TODO: Debug why ^ is failing with flaky differences on Github Actions, see https://pastebin.com/BUa1btnh
+	tester.RunInteractiveShell(t, ` HISHTORY_REDACT_FORCE=true hishtory redact sleep 13371337`)
+
+	// And confirm it was redacted
+	out = tester.RunInteractiveShell(t, ` hishtory export sleep -export`)
+	require.Equal(t, "", out)
+
+	// Then go back to device1 and confirm it was redacted there too
+	restoreDevice1()
+	out = tester.RunInteractiveShell(t, ` hishtory export sleep -export`)
+	require.Equal(t, "", out)
 }
 
 func testUninstall(t *testing.T, tester shellTester) {

client/cmd/install.go

@@ -171,9 +171,19 @@ func install(secretKey string, offline bool) error {
 		// No config, so set up a new installation
 		return lib.Setup(secretKey, offline)
 	}
+	err = handleDbUpgrades(hctx.MakeContext())
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
+// Handles people running `hishtory update` when the DB needs updating.
+func handleDbUpgrades(ctx context.Context) error {
+	db := hctx.GetDb(ctx)
+	return db.Exec(`UPDATE history_entries SET entry_id = lower(hex(randomblob(12))) WHERE entry_id IS NULL`).Error
+}
+
 // Handles people running `hishtory update` from an old version of hishtory that
 // doesn't support the control-r integration, so that they'll get control-r enabled
 // but someone who has it explicitly disabled will keep it that way.

client/cmd/redact.go

@@ -86,7 +86,9 @@ func deleteOnRemoteInstances(ctx context.Context, historyEntries []*data.History
 	deletionRequest.UserId = data.UserId(config.UserSecret)
 
 	for _, entry := range historyEntries {
-		deletionRequest.Messages.Ids = append(deletionRequest.Messages.Ids, shared.MessageIdentifier{Date: entry.EndTime, DeviceId: entry.DeviceId})
+		deletionRequest.Messages.Ids = append(deletionRequest.Messages.Ids,
+			shared.MessageIdentifier{DeviceId: entry.DeviceId, EndTime: entry.EndTime, EntryId: entry.EntryId},
+		)
 	}
 	return lib.SendDeletionRequest(deletionRequest)
 }

client/cmd/saveHistoryEntry.go

@@ -18,7 +18,9 @@ import (
 	"github.com/ddworken/hishtory/client/hctx"
 	"github.com/ddworken/hishtory/client/lib"
 	"github.com/ddworken/hishtory/shared"
+	"github.com/google/uuid"
 	"github.com/spf13/cobra"
+	"gorm.io/gorm"
 )
 
 var saveHistoryEntryCmd = &cobra.Command{
@@ -81,6 +83,23 @@ func maybeUploadSkippedHistoryEntries(ctx context.Context) error {
 	return nil
 }
 
+func handlePotentialUploadFailure(err error, config *hctx.ClientConfig, entryTimestamp time.Time) {
+	if err != nil {
+		if lib.IsOfflineError(err) {
+			hctx.GetLogger().Infof("Failed to remotely persist hishtory entry because we failed to connect to the remote server! This is likely because the device is offline, but also could be because the remote server is having reliability issues. Original error: %v", err)
+			if !config.HaveMissedUploads {
+				config.HaveMissedUploads = true
+				// Set MissedUploadTimestamp to `entry timestamp - 1` so that the current entry will get
+				// uploaded once network access is regained.
+				config.MissedUploadTimestamp = entryTimestamp.UTC().Unix() - 1
+				lib.CheckFatalError(hctx.SetConfig(*config))
+			}
+		} else {
+			lib.CheckFatalError(err)
+		}
+	}
+}
+
 func presaveHistoryEntry(ctx context.Context) {
 	config := hctx.GetConf(ctx)
 	if !config.IsEnabled {
@@ -119,12 +138,13 @@ func presaveHistoryEntry(ctx context.Context) {
 	lib.CheckFatalError(err)
 	db.Commit()
 
-	// Note that we aren't persisting these half-entries remotely,
+	// And persist it remotely
-	// since they should be updated with the rest of the information very soon. If they
+	if !config.IsOffline {
-	// are never updated (e.g. due to a long-running command that never finishes), then
+		jsonValue, err := lib.EncryptAndMarshal(config, []*data.HistoryEntry{entry})
-	// they will only be available on this device. That isn't perfect since it means
+		lib.CheckFatalError(err)
-	// history entries can get out of sync, but it is probably good enough.
+		_, err = lib.ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
-	// TODO: Consider improving this
+		handlePotentialUploadFailure(err, &config, entry.StartTime)
+	}
 }
 
 func saveHistoryEntry(ctx context.Context) {
@@ -143,25 +163,7 @@ func saveHistoryEntry(ctx context.Context) {
 
 	// Drop any entries from pre-saving since they're no longer needed
 	if config.BetaMode {
-		deletePresavedEntryFunc := func() error {
+		lib.CheckFatalError(deletePresavedEntries(ctx, entry))
-			query := "cwd:" + entry.CurrentWorkingDirectory
-			query += " start_time:" + strconv.FormatInt(entry.StartTime.Unix(), 10)
-			query += " end_time:1970/01/01_00:00:00_+00:00"
-			tx, err := lib.MakeWhereQueryFromSearch(ctx, db, query)
-			if err != nil {
-				return fmt.Errorf("failed to query for pre-saved history entry: %w", err)
-			}
-			tx.Where("command = ?", entry.Command)
-			res := tx.Delete(&data.HistoryEntry{})
-			if res.Error != nil {
-				return fmt.Errorf("failed to delete pre-saved history entry (expected command=%#v): %w", entry.Command, res.Error)
-			}
-			if res.RowsAffected > 1 {
-				return fmt.Errorf("attempted to delete pre-saved entry, but something went wrong since we deleted %d rows", res.RowsAffected)
-			}
-			return nil
-		}
-		lib.CheckFatalError(lib.RetryingDbFunction(deletePresavedEntryFunc))
 	}
 
 	// Persist it locally
@@ -169,82 +171,105 @@ func saveHistoryEntry(ctx context.Context) {
 	lib.CheckFatalError(err)
 
 	// Persist it remotely
-	shouldCheckForDeletionRequests := true
-	shouldCheckForDumpRequests := true
 	if !config.IsOffline {
 		jsonValue, err := lib.EncryptAndMarshal(config, []*data.HistoryEntry{entry})
 		lib.CheckFatalError(err)
 		w, err := lib.ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
+		handlePotentialUploadFailure(err, &config, entry.StartTime)
 		if err == nil {
 			submitResponse := shared.SubmitResponse{}
 			err := json.Unmarshal(w, &submitResponse)
 			if err != nil {
 				lib.CheckFatalError(fmt.Errorf("failed to deserialize response from /api/v1/submit: %w", err))
 			}
-			shouldCheckForDeletionRequests = submitResponse.HaveDeletionRequests
+			lib.CheckFatalError(lib.HandleDeletionRequests(ctx, submitResponse.DeletionRequests))
-			shouldCheckForDumpRequests = submitResponse.HaveDumpRequests
+			lib.CheckFatalError(handleDumpRequests(ctx, submitResponse.DumpRequests))
-		} else {
-			if lib.IsOfflineError(err) {
-				hctx.GetLogger().Infof("Failed to remotely persist hishtory entry because we failed to connect to the remote server! This is likely because the device is offline, but also could be because the remote server is having reliability issues. Original error: %v", err)
-				if !config.HaveMissedUploads {
-					config.HaveMissedUploads = true
-					config.MissedUploadTimestamp = time.Now().Unix()
-					lib.CheckFatalError(hctx.SetConfig(config))
-				}
-			} else {
-				lib.CheckFatalError(err)
-			}
 		}
 	}
 
-	// Check if there is a pending dump request and reply to it if so
-	if shouldCheckForDumpRequests {
-		dumpRequests, err := lib.GetDumpRequests(config)
-		if err != nil {
-			if lib.IsOfflineError(err) {
-				// It is fine to just ignore this, the next command will retry the API and eventually we will respond to any pending dump requests
-				dumpRequests = []*shared.DumpRequest{}
-				hctx.GetLogger().Infof("Failed to check for dump requests because we failed to connect to the remote server!")
-			} else {
-				lib.CheckFatalError(err)
-			}
-		}
-		if len(dumpRequests) > 0 {
-			lib.CheckFatalError(lib.RetrieveAdditionalEntriesFromRemote(ctx))
-			entries, err := lib.Search(ctx, db, "", 0)
-			lib.CheckFatalError(err)
-			var encEntries []*shared.EncHistoryEntry
-			for _, entry := range entries {
-				enc, err := data.EncryptHistoryEntry(config.UserSecret, *entry)
-				lib.CheckFatalError(err)
-				encEntries = append(encEntries, &enc)
-			}
-			reqBody, err := json.Marshal(encEntries)
-			lib.CheckFatalError(err)
-			for _, dumpRequest := range dumpRequests {
-				if !config.IsOffline {
-					_, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId+"&source_device_id="+config.DeviceId, "application/json", reqBody)
-					lib.CheckFatalError(err)
-				}
-			}
-		}
-	}
-
-	// Handle deletion requests
-	if shouldCheckForDeletionRequests {
-		lib.CheckFatalError(lib.ProcessDeletionRequests(ctx))
-	}
-
 	if config.BetaMode {
 		db.Commit()
 	}
 }
 
+func deletePresavedEntries(ctx context.Context, entry *data.HistoryEntry) error {
+	db := hctx.GetDb(ctx)
+
+	// Create the query to find the presaved entries
+	query := "cwd:" + entry.CurrentWorkingDirectory
+	query += " start_time:" + strconv.FormatInt(entry.StartTime.Unix(), 10)
+	query += " end_time:1970/01/01_00:00:00_+00:00"
+	matchingEntryQuery, err := lib.MakeWhereQueryFromSearch(ctx, db, query)
+	if err != nil {
+		return fmt.Errorf("failed to query for pre-saved history entry: %w", err)
+	}
+	matchingEntryQuery = matchingEntryQuery.Where("command = ?", entry.Command).Session(&gorm.Session{})
+
+	// Get the presaved entry since we need it for doing remote deletes
+	var presavedEntry data.HistoryEntry
+	res := matchingEntryQuery.Find(&presavedEntry)
+	if res.Error != nil {
+		return fmt.Errorf("failed to search for presaved entry for cmd=%#v: %w", entry.Command, res.Error)
+	}
+
+	// Delete presaved entries locally
+	deletePresavedEntryFunc := func() error {
+		res := matchingEntryQuery.Delete(&data.HistoryEntry{})
+		if res.Error != nil {
+			return fmt.Errorf("failed to delete pre-saved history entry (expected command=%#v): %w", entry.Command, res.Error)
+		}
+		if res.RowsAffected > 1 {
+			return fmt.Errorf("attempted to delete pre-saved entry, but something went wrong since we deleted %d rows", res.RowsAffected)
+		}
+		return nil
+	}
+	err = lib.RetryingDbFunction(deletePresavedEntryFunc)
+	if err != nil {
+		return err
+	}
+
+	// And delete it remotely
+	config := hctx.GetConf(ctx)
+	var deletionRequest shared.DeletionRequest
+	deletionRequest.SendTime = time.Now()
+	deletionRequest.UserId = data.UserId(config.UserSecret)
+	deletionRequest.Messages.Ids = append(deletionRequest.Messages.Ids,
+		// Note that we aren't specifying an EndTime here since pre-saved entries don't have an EndTime
+		shared.MessageIdentifier{DeviceId: presavedEntry.DeviceId, EntryId: presavedEntry.EntryId},
+	)
+	return lib.SendDeletionRequest(deletionRequest)
+}
+
 func init() {
 	rootCmd.AddCommand(saveHistoryEntryCmd)
 	rootCmd.AddCommand(presaveHistoryEntryCmd)
 }
 
+func handleDumpRequests(ctx context.Context, dumpRequests []*shared.DumpRequest) error {
+	db := hctx.GetDb(ctx)
+	config := hctx.GetConf(ctx)
+	if len(dumpRequests) > 0 {
+		lib.CheckFatalError(lib.RetrieveAdditionalEntriesFromRemote(ctx))
+		entries, err := lib.Search(ctx, db, "", 0)
+		lib.CheckFatalError(err)
+		var encEntries []*shared.EncHistoryEntry
+		for _, entry := range entries {
+			enc, err := data.EncryptHistoryEntry(config.UserSecret, *entry)
+			lib.CheckFatalError(err)
+			encEntries = append(encEntries, &enc)
+		}
+		reqBody, err := json.Marshal(encEntries)
+		lib.CheckFatalError(err)
+		for _, dumpRequest := range dumpRequests {
+			if !config.IsOffline {
+				_, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId+"&source_device_id="+config.DeviceId, "application/json", reqBody)
+				lib.CheckFatalError(err)
+			}
+		}
+	}
+	return nil
+}
+
 func buildPreArgsHistoryEntry(ctx context.Context) (*data.HistoryEntry, error) {
 	var entry data.HistoryEntry
 
@@ -274,6 +299,9 @@ func buildPreArgsHistoryEntry(ctx context.Context) (*data.HistoryEntry, error) {
 	config := hctx.GetConf(ctx)
 	entry.DeviceId = config.DeviceId
 
+	// entry ID
+	entry.EntryId = uuid.Must(uuid.NewRandom()).String()
+
 	// custom columns
 	cc, err := buildCustomColumns(ctx)
 	if err != nil {

client/cmd/status.go

@@ -22,7 +22,6 @@ var statusCmd = &cobra.Command{
 		if *verbose {
 			fmt.Printf("User ID: %s\n", data.UserId(config.UserSecret))
 			fmt.Printf("Device ID: %s\n", config.DeviceId)
-			printDumpStatus(config)
 			printOnlineStatus(config)
 		}
 		fmt.Printf("Commit Hash: %s\n", lib.GitCommit)
@@ -42,16 +41,6 @@ func printOnlineStatus(config hctx.ClientConfig) {
 	}
 }
 
-func printDumpStatus(config hctx.ClientConfig) {
-	dumpRequests, err := lib.GetDumpRequests(config)
-	lib.CheckFatalError(err)
-	fmt.Printf("Dump Requests: ")
-	for _, d := range dumpRequests {
-		fmt.Printf("%#v, ", *d)
-	}
-	fmt.Print("\n")
-}
-
 func init() {
 	rootCmd.AddCommand(statusCmd)
 	verbose = statusCmd.Flags().BoolP("verbose", "v", false, "Display verbose hiSHtory information")

client/data/data.go

@@ -15,7 +15,6 @@ import (
 	"time"
 
 	"github.com/ddworken/hishtory/shared"
-	"github.com/google/uuid"
 )
 
 const (
@@ -39,6 +38,7 @@ type HistoryEntry struct {
 	StartTime               time.Time     `json:"start_time" gorm:"uniqueIndex:compositeindex"`
 	EndTime                 time.Time     `json:"end_time" gorm:"uniqueIndex:compositeindex,index:end_time_index"`
 	DeviceId                string        `json:"device_id" gorm:"uniqueIndex:compositeindex"`
+	EntryId                 string        `json:"entry_id" gorm:"uniqueIndex:compositeindex,uniqueIndex:entry_id_index"`
 	CustomColumns           CustomColumns `json:"custom_columns"`
 }
 
@@ -136,7 +136,7 @@ func EncryptHistoryEntry(userSecret string, entry HistoryEntry) (shared.EncHisto
 		Nonce:         nonce,
 		UserId:        UserId(userSecret),
 		Date:          entry.EndTime,
-		EncryptedId:   uuid.Must(uuid.NewRandom()).String(),
+		EncryptedId:   entry.EntryId,
 		ReadCount:     0,
 	}, nil
 }

client/hctx/hctx.go

@@ -103,6 +103,7 @@ func OpenLocalSqliteDb() (*gorm.DB, error) {
 	db.AutoMigrate(&data.HistoryEntry{})
 	db.Exec("PRAGMA journal_mode = WAL")
 	db.Exec("CREATE INDEX IF NOT EXISTS end_time_index ON history_entries(end_time)")
+	db.Exec("CREATE INDEX IF NOT EXISTS entry_id_index ON history_entries(entry_id)")
 	return db, nil
 }
 

client/lib/lib.go

@@ -303,6 +303,7 @@ func ImportHistory(ctx context.Context, shouldReadStdin, force bool) (int, error
 			StartTime:               time.Now().UTC(),
 			EndTime:                 time.Now().UTC(),
 			DeviceId:                config.DeviceId,
+			EntryId:                 uuid.Must(uuid.NewRandom()).String(),
 		}
 		err = ReliableDbCreate(db, entry)
 		if err != nil {
@@ -597,10 +598,17 @@ func ProcessDeletionRequests(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	return HandleDeletionRequests(ctx, deletionRequests)
+}
+
+func HandleDeletionRequests(ctx context.Context, deletionRequests []*shared.DeletionRequest) error {
 	db := hctx.GetDb(ctx)
 	for _, request := range deletionRequests {
 		for _, entry := range request.Messages.Ids {
-			res := db.Where("device_id = ? AND end_time = ?", entry.DeviceId, entry.Date).Delete(&data.HistoryEntry{})
+			// Note that entry.EndTime is not always present (for pre-saved entries). And likewise,
+			// entry.EntryId is not always present for older entries. So we just check that one of them matches.
+			tx := db.Where("device_id = ? AND (end_time = ? OR entry_id = ?)", entry.DeviceId, entry.EndTime, entry.EntryId)
+			res := tx.Delete(&data.HistoryEntry{})
 			if res.Error != nil {
 				return fmt.Errorf("DB error: %w", res.Error)
 			}
@@ -872,22 +880,6 @@ func unescape(query string) string {
 	return string(newQuery)
 }
 
-func GetDumpRequests(config hctx.ClientConfig) ([]*shared.DumpRequest, error) {
-	if config.IsOffline {
-		return make([]*shared.DumpRequest, 0), nil
-	}
-	resp, err := ApiGet("/api/v1/get-dump-requests?user_id=" + data.UserId(config.UserSecret) + "&device_id=" + config.DeviceId)
-	if IsOfflineError(err) {
-		return []*shared.DumpRequest{}, nil
-	}
-	if err != nil {
-		return nil, err
-	}
-	var dumpRequests []*shared.DumpRequest
-	err = json.Unmarshal(resp, &dumpRequests)
-	return dumpRequests, err
-}
-
 func SendDeletionRequest(deletionRequest shared.DeletionRequest) error {
 	data, err := json.Marshal(deletionRequest)
 	if err != nil {

client/tui/tui.go

@@ -601,7 +601,9 @@ func deleteHistoryEntry(ctx context.Context, entry data.HistoryEntry) error {
 		UserId:   data.UserId(hctx.GetConf(ctx).UserSecret),
 		SendTime: time.Now(),
 	}
-	dr.Messages.Ids = append(dr.Messages.Ids, shared.MessageIdentifier{Date: entry.EndTime, DeviceId: entry.DeviceId})
+	dr.Messages.Ids = append(dr.Messages.Ids,
+		shared.MessageIdentifier{DeviceId: entry.DeviceId, EndTime: entry.EndTime, EntryId: entry.EntryId},
+	)
 	return lib.SendDeletionRequest(dr)
 }
 

shared/data.go

@@ -9,22 +9,17 @@ import (
 
 // Represents an encrypted history entry
 type EncHistoryEntry struct {
-	EncryptedData []byte    `json:"enc_data"`
+	EncryptedData []byte `json:"enc_data"`
-	Nonce         []byte    `json:"nonce"`
+	Nonce         []byte `json:"nonce"`
-	DeviceId      string    `json:"device_id"`
+	DeviceId      string `json:"device_id"`
-	UserId        string    `json:"user_id"`
+	UserId        string `json:"user_id"`
-	Date          time.Time `json:"time"`
+	// Note that EncHistoryEntry.Date == HistoryEntry.EndTime
-	EncryptedId   string    `json:"id"`
+	Date time.Time `json:"time"`
-	ReadCount     int       `json:"read_count"`
+	// Note that EncHistoryEntry.EncryptedId == HistoryEntry.Id (for entries created after pre-saving support)
+	EncryptedId string `json:"encrypted_id"`
+	ReadCount   int    `json:"read_count"`
 }
 
-/*
-Manually created the indices:
-CREATE INDEX CONCURRENTLY device_id_idx ON enc_history_entries USING btree(device_id);
-CREATE INDEX CONCURRENTLY read_count_idx ON enc_history_entries USING btree(read_count);
-CREATE INDEX CONCURRENTLY redact_idx ON enc_history_entries USING btree(user_id, device_id, date);
-*/
-
 type Device struct {
 	UserId   string `json:"user_id"`
 	DeviceId string `json:"device_id"`
@@ -81,14 +76,19 @@ type MessageIdentifiers struct {
 	Ids []MessageIdentifier `json:"message_ids"`
 }
 
-// Identifies a single history entry based on the device that recorded the entry, and the end time. Note that
+// Identifies a single history entry based on the device that recorded the entry, and additional metadata. Note that
 // this does not include the command itself since that would risk including the sensitive data that is meant
 // to be deleted
 type MessageIdentifier struct {
 	// The device that the entry was recorded on (NOT the device where it is stored/requesting deletion)
 	DeviceId string `json:"device_id"`
-	// The timestamp when the message finished running
+	// The timestamp when the command finished running. Serialized as "date" for legacy compatibility.
-	Date time.Time `json:"date"`
+	EndTime time.Time `json:"date"`
+	// The entry ID of the command.
+	// Note this field was added as part of supporting pre-saving commands, so older clients do not set this field
+	// And even for new clients, it may contain a per-device entry ID. For pre-saved entries, this is guaranteed to
+	// be present.
+	EntryId string `json:"entry_id"`
 }
 
 func (m *MessageIdentifiers) Scan(value interface{}) error {
@@ -114,11 +114,11 @@ type Feedback struct {
 	Feedback string    `json:"feedback"`
 }
 
-// Response from submitting new history entries. Contains metadata that is used to avoid making additional round-trip
+// Response from submitting new history entries. Contains deletion requests and dump requests to avoid
-// requests to the hishtory backend.
+// extra round-trip requests to the hishtory backend.
 type SubmitResponse struct {
-	HaveDumpRequests     bool `json:"have_dump_requests"`
+	DumpRequests     []*DumpRequest     `json:"dump_requests"`
-	HaveDeletionRequests bool `json:"have_deletion_requests"`
+	DeletionRequests []*DeletionRequest `json:"deletion_requests"`
 }
 
 func Chunks[k any](slice []k, chunkSize int) [][]k {

shared/testutils/testutils.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/ddworken/hishtory/client/data"
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
 )
 
 const (
@@ -326,6 +327,7 @@ func MakeFakeHistoryEntry(command string) data.HistoryEntry {
 		StartTime:               time.Unix(fakeHistoryTimestamp, 0).UTC(),
 		EndTime:                 time.Unix(fakeHistoryTimestamp+3, 0).UTC(),
 		DeviceId:                "fake_device_id",
+		EntryId:                 uuid.Must(uuid.NewRandom()).String(),
 	}
 }