extern/hishtory
1
0
Fork 0
mirror of https://github.com/ddworken/hishtory.git synced 2024-11-23 00:34:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #111 from ddworken/sync-bug

Browse Source 
Fix bug in offline syncing resumption
This commit is contained in:
David Dworken 2023-09-22 19:35:38 -07:00 committed by GitHub
commit 8bb02ea88c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 226 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
backend/server/internal/database/db.go
View File

@ -63,6 +63,26 @@ func (db *DB) AddDatabaseTables() error {
return nil
}
func (db *DB) CreateIndices() error {
// Note: If adding a new index here, consider manually running it on the prod DB using CONCURRENTLY to
// make server startup non-blocking. The benefit of this function is primarily for other people so they
// don't have to manually create these indexes.
var indices = []string{
`CREATE INDEX IF NOT EXISTS entry_id_idx ON enc_history_entries USING btree(encrypted_id);`,
`CREATE INDEX IF NOT EXISTS device_id_idx ON enc_history_entries USING btree(device_id);`,
`CREATE INDEX IF NOT EXISTS read_count_idx ON enc_history_entries USING btree(read_count);`,
`CREATE INDEX IF NOT EXISTS redact_idx ON enc_history_entries USING btree(user_id, device_id, date);`,
`CREATE INDEX IF NOT EXISTS del_user_idx ON deletion_requests USING btree(user_id);`,
}
for _, index := range indices {
r := db.Exec(index)
if r.Error != nil {
return fmt.Errorf("failed to execute index creation sql=%#v: %w", index, r.Error)
}
}
return nil
}
func (db *DB) Close() error {
rawDB, err := db.DB.DB()
if err != nil {
@ -152,7 +172,8 @@ func (db *DB) DumpRequestDeleteForUserAndDevice(ctx context.Context, userID, dev
func (db *DB) ApplyDeletionRequestsToBackend(ctx context.Context, request *shared.DeletionRequest) (int64, error) {
tx := db.WithContext(ctx).Where("false")
for _, message := range request.Messages.Ids {
tx = tx.Or(db.WithContext(ctx).Where("user_id = ? AND device_id = ? AND date = ?", request.UserId, message.DeviceId, message.Date))
// Note that we do an OR with date or the ID matching since the ID is not always recorded for older history entries.
tx = tx.Or(db.WithContext(ctx).Where("user_id = ? AND device_id = ? AND (date = ? OR encrypted_id = ?)", request.UserId, message.DeviceId, message.EndTime, message.EntryId))
}
result := tx.Delete(&shared.EncHistoryEntry{})
if tx.Error != nil {

33
backend/server/internal/server/api_handlers.go
View File

@ -1,7 +1,6 @@
package server
import (
"context"
"encoding/json"
"fmt"
"html"
@ -49,34 +48,24 @@ func (s *Server) apiSubmitHandler(w http.ResponseWriter, r *http.Request) {
s.statsd.Count("hishtory.submit", int64(len(devices)), []string{}, 1.0)
}
resp := shared.SubmitResponse{}
deviceId := getOptionalQueryParam(r, "source_device_id", s.isTestEnvironment)
resp := shared.SubmitResponse{
HaveDumpRequests: s.haveDumpRequests(r.Context(), userId, deviceId),
HaveDeletionRequests: s.haveDeletionRequests(r.Context(), userId, deviceId),
if deviceId != "" {
dumpRequests, err := s.db.DumpRequestForUserAndDevice(r.Context(), userId, deviceId)
checkGormError(err)
resp.DumpRequests = dumpRequests
deletionRequests, err := s.db.DeletionRequestsForUserAndDevice(r.Context(), userId, deviceId)
checkGormError(err)
resp.DeletionRequests = deletionRequests
}
if err := json.NewEncoder(w).Encode(resp); err != nil {
panic(err)
}
}
func (s *Server) haveDumpRequests(ctx context.Context, userId, deviceId string) bool {
if userId == "" || deviceId == "" {
return true
}
dumpRequests, err := s.db.DumpRequestForUserAndDevice(ctx, userId, deviceId)
checkGormError(err)
return len(dumpRequests) > 0
}
func (s *Server) haveDeletionRequests(ctx context.Context, userId, deviceId string) bool {
if userId == "" || deviceId == "" {
return true
}
deletionRequests, err := s.db.DeletionRequestsForUserAndDevice(ctx, userId, deviceId)
checkGormError(err)
return len(deletionRequests) > 0
}
func (s *Server) apiBootstrapHandler(w http.ResponseWriter, r *http.Request) {
userId := getRequiredQueryParam(r, "user_id")
deviceId := getRequiredQueryParam(r, "device_id")

22
backend/server/internal/server/server_test.go
View File

@ -81,7 +81,8 @@ func TestESubmitThenQuery(t *testing.T) {
w := httptest.NewRecorder()
s.apiSubmitHandler(w, submitReq)
require.Equal(t, 200, w.Result().StatusCode)
require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: false}, deserializeSubmitResponse(t, w))
require.Empty(t, deserializeSubmitResponse(t, w).DeletionRequests)
require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
// Query for device id 1
w = httptest.NewRecorder()
@ -346,7 +347,8 @@ func TestDeletionRequests(t *testing.T) {
w := httptest.NewRecorder()
s.apiSubmitHandler(w, submitReq)
require.Equal(t, 200, w.Result().StatusCode)
require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: false}, deserializeSubmitResponse(t, w))
require.Empty(t, deserializeSubmitResponse(t, w).DeletionRequests)
require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
// And another entry for user1
entry2 := testutils.MakeFakeHistoryEntry("ls /foo/bar")
@ -359,7 +361,8 @@ func TestDeletionRequests(t *testing.T) {
w = httptest.NewRecorder()
s.apiSubmitHandler(w, submitReq)
require.Equal(t, 200, w.Result().StatusCode)
require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: false}, deserializeSubmitResponse(t, w))
require.Empty(t, deserializeSubmitResponse(t, w).DeletionRequests)
require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
// And an entry for user2 that has the same timestamp as the previous entry
entry3 := testutils.MakeFakeHistoryEntry("ls /foo/bar")
@ -373,7 +376,8 @@ func TestDeletionRequests(t *testing.T) {
w = httptest.NewRecorder()
s.apiSubmitHandler(w, submitReq)
require.Equal(t, 200, w.Result().StatusCode)
require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: false}, deserializeSubmitResponse(t, w))
require.Empty(t, deserializeSubmitResponse(t, w).DeletionRequests)
require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
// Query for device id 1
w = httptest.NewRecorder()
@ -411,7 +415,7 @@ func TestDeletionRequests(t *testing.T) {
UserId: data.UserId("dkey"),
SendTime: delReqTime,
Messages: shared.MessageIdentifiers{Ids: []shared.MessageIdentifier{
{DeviceId: devId1, Date: entry1.EndTime},
{DeviceId: devId1, EndTime: entry1.EndTime},
}},
}
reqBody, err = json.Marshal(delReq)
@ -485,7 +489,8 @@ func TestDeletionRequests(t *testing.T) {
w = httptest.NewRecorder()
s.apiSubmitHandler(w, submitReq)
require.Equal(t, 200, w.Result().StatusCode)
require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: true}, deserializeSubmitResponse(t, w))
require.NotEmpty(t, deserializeSubmitResponse(t, w).DeletionRequests)
require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
// Query for deletion requests
w = httptest.NewRecorder()
@ -507,7 +512,7 @@ func TestDeletionRequests(t *testing.T) {
SendTime: delReqTime,
ReadCount: 1,
Messages: shared.MessageIdentifiers{Ids: []shared.MessageIdentifier{
{DeviceId: devId1, Date: entry1.EndTime},
{DeviceId: devId1, EndTime: entry1.EndTime},
}},
}
if diff := deep.Equal(*deletionRequest, expected); diff != nil {
@ -585,7 +590,8 @@ func TestCleanDatabaseNoErrors(t *testing.T) {
w := httptest.NewRecorder()
s.apiSubmitHandler(w, submitReq)
require.Equal(t, 200, w.Result().StatusCode)
require.Equal(t, shared.SubmitResponse{HaveDumpRequests: true, HaveDeletionRequests: false}, deserializeSubmitResponse(t, w))
require.Empty(t, deserializeSubmitResponse(t, w).DeletionRequests)
require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests)
// Call cleanDatabase and just check that there are no panics
testutils.Check(t, DB.Clean(context.TODO()))

4
backend/server/server.go
View File

@ -95,6 +95,10 @@ func OpenDB() (*database.DB, error) {
return nil, fmt.Errorf("failed to create underlying DB tables: %w", err)
}
}
err := db.CreateIndices()
if err != nil {
return nil, err
}
return db, nil
}

27
client/client_test.go
View File

@ -2364,15 +2364,24 @@ func testPresaving(t *testing.T, tester shellTester) {
out = tester.RunInteractiveShell(t, ` hishtory query sleep 13371337 -export -tquery`)
testutils.CompareGoldens(t, out, "testPresaving-query")
// And the same for tquery
// out = captureTerminalOutputWithComplexCommands(t, tester,
// []TmuxCommand{
// {Keys: "hishtory SPACE tquery ENTER", ExtraDelay: 2.0},
// {Keys: "sleep SPACE 13371337 SPACE -export SPACE -tquery"}})
// out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1])
// testutils.CompareGoldens(t, out, "testPresaving-tquery")
//
// TODO: Debug why ^ is failing with flaky differences on Github Actions, see https://pastebin.com/BUa1btnh
// Create a new device, and confirm it shows up there too
restoreDevice1 := testutils.BackupAndRestoreWithId(t, "device1")
installHishtory(t, tester, userSecret)
tester.RunInteractiveShell(t, ` hishtory config-set displayed-columns CWD Runtime Command`)
out = tester.RunInteractiveShell(t, ` hishtory query sleep 13371337 -export -tquery`)
testutils.CompareGoldens(t, out, "testPresaving-query")
// And then redact it from device2
tester.RunInteractiveShell(t, ` HISHTORY_REDACT_FORCE=true hishtory redact sleep 13371337`)
// And confirm it was redacted
out = tester.RunInteractiveShell(t, ` hishtory export sleep -export`)
require.Equal(t, "", out)
// Then go back to device1 and confirm it was redacted there too
restoreDevice1()
out = tester.RunInteractiveShell(t, ` hishtory export sleep -export`)
require.Equal(t, "", out)
}
func testUninstall(t *testing.T, tester shellTester) {

10
client/cmd/install.go
View File

@ -171,9 +171,19 @@ func install(secretKey string, offline bool) error {
// No config, so set up a new installation
return lib.Setup(secretKey, offline)
}
err = handleDbUpgrades(hctx.MakeContext())
if err != nil {
return err
}
return nil
}
// Handles people running `hishtory update` when the DB needs updating.
func handleDbUpgrades(ctx context.Context) error {
db := hctx.GetDb(ctx)
return db.Exec(`UPDATE history_entries SET entry_id = lower(hex(randomblob(12))) WHERE entry_id IS NULL`).Error
}
// Handles people running `hishtory update` from an old version of hishtory that
// doesn't support the control-r integration, so that they'll get control-r enabled
// but someone who has it explicitly disabled will keep it that way.

4
client/cmd/redact.go
View File

@ -86,7 +86,9 @@ func deleteOnRemoteInstances(ctx context.Context, historyEntries []*data.History
deletionRequest.UserId = data.UserId(config.UserSecret)
for _, entry := range historyEntries {
deletionRequest.Messages.Ids = append(deletionRequest.Messages.Ids, shared.MessageIdentifier{Date: entry.EndTime, DeviceId: entry.DeviceId})
deletionRequest.Messages.Ids = append(deletionRequest.Messages.Ids,
shared.MessageIdentifier{DeviceId: entry.DeviceId, EndTime: entry.EndTime, EntryId: entry.EntryId},
)
}
return lib.SendDeletionRequest(deletionRequest)
}

184
client/cmd/saveHistoryEntry.go
View File

@ -18,7 +18,9 @@ import (
"github.com/ddworken/hishtory/client/hctx"
"github.com/ddworken/hishtory/client/lib"
"github.com/ddworken/hishtory/shared"
"github.com/google/uuid"
"github.com/spf13/cobra"
"gorm.io/gorm"
)
var saveHistoryEntryCmd = &cobra.Command{
@ -81,6 +83,23 @@ func maybeUploadSkippedHistoryEntries(ctx context.Context) error {
return nil
}
func handlePotentialUploadFailure(err error, config *hctx.ClientConfig, entryTimestamp time.Time) {
if err != nil {
if lib.IsOfflineError(err) {
hctx.GetLogger().Infof("Failed to remotely persist hishtory entry because we failed to connect to the remote server! This is likely because the device is offline, but also could be because the remote server is having reliability issues. Original error: %v", err)
if !config.HaveMissedUploads {
config.HaveMissedUploads = true
// Set MissedUploadTimestamp to `entry timestamp - 1` so that the current entry will get
// uploaded once network access is regained.
config.MissedUploadTimestamp = entryTimestamp.UTC().Unix() - 1
lib.CheckFatalError(hctx.SetConfig(*config))
}
} else {
lib.CheckFatalError(err)
}
}
}
func presaveHistoryEntry(ctx context.Context) {
config := hctx.GetConf(ctx)
if !config.IsEnabled {
@ -119,12 +138,13 @@ func presaveHistoryEntry(ctx context.Context) {
lib.CheckFatalError(err)
db.Commit()
// Note that we aren't persisting these half-entries remotely,
// since they should be updated with the rest of the information very soon. If they
// are never updated (e.g. due to a long-running command that never finishes), then
// they will only be available on this device. That isn't perfect since it means
// history entries can get out of sync, but it is probably good enough.
// TODO: Consider improving this
// And persist it remotely
if !config.IsOffline {
jsonValue, err := lib.EncryptAndMarshal(config, []*data.HistoryEntry{entry})
lib.CheckFatalError(err)
_, err = lib.ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
handlePotentialUploadFailure(err, &config, entry.StartTime)
}
}
func saveHistoryEntry(ctx context.Context) {
@ -143,25 +163,7 @@ func saveHistoryEntry(ctx context.Context) {
// Drop any entries from pre-saving since they're no longer needed
if config.BetaMode {
deletePresavedEntryFunc := func() error {
query := "cwd:" + entry.CurrentWorkingDirectory
query += " start_time:" + strconv.FormatInt(entry.StartTime.Unix(), 10)
query += " end_time:1970/01/01_00:00:00_+00:00"
tx, err := lib.MakeWhereQueryFromSearch(ctx, db, query)
if err != nil {
return fmt.Errorf("failed to query for pre-saved history entry: %w", err)
}
tx.Where("command = ?", entry.Command)
res := tx.Delete(&data.HistoryEntry{})
if res.Error != nil {
return fmt.Errorf("failed to delete pre-saved history entry (expected command=%#v): %w", entry.Command, res.Error)
}
if res.RowsAffected > 1 {
return fmt.Errorf("attempted to delete pre-saved entry, but something went wrong since we deleted %d rows", res.RowsAffected)
}
return nil
}
lib.CheckFatalError(lib.RetryingDbFunction(deletePresavedEntryFunc))
lib.CheckFatalError(deletePresavedEntries(ctx, entry))
}
// Persist it locally
@ -169,82 +171,105 @@ func saveHistoryEntry(ctx context.Context) {
lib.CheckFatalError(err)
// Persist it remotely
shouldCheckForDeletionRequests := true
shouldCheckForDumpRequests := true
if !config.IsOffline {
jsonValue, err := lib.EncryptAndMarshal(config, []*data.HistoryEntry{entry})
lib.CheckFatalError(err)
w, err := lib.ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
handlePotentialUploadFailure(err, &config, entry.StartTime)
if err == nil {
submitResponse := shared.SubmitResponse{}
err := json.Unmarshal(w, &submitResponse)
if err != nil {
lib.CheckFatalError(fmt.Errorf("failed to deserialize response from /api/v1/submit: %w", err))
}
shouldCheckForDeletionRequests = submitResponse.HaveDeletionRequests
shouldCheckForDumpRequests = submitResponse.HaveDumpRequests
} else {
if lib.IsOfflineError(err) {
hctx.GetLogger().Infof("Failed to remotely persist hishtory entry because we failed to connect to the remote server! This is likely because the device is offline, but also could be because the remote server is having reliability issues. Original error: %v", err)
if !config.HaveMissedUploads {
config.HaveMissedUploads = true
config.MissedUploadTimestamp = time.Now().Unix()
lib.CheckFatalError(hctx.SetConfig(config))
}
} else {
lib.CheckFatalError(err)
}
lib.CheckFatalError(lib.HandleDeletionRequests(ctx, submitResponse.DeletionRequests))
lib.CheckFatalError(handleDumpRequests(ctx, submitResponse.DumpRequests))
}
}
// Check if there is a pending dump request and reply to it if so
if shouldCheckForDumpRequests {
dumpRequests, err := lib.GetDumpRequests(config)
if err != nil {
if lib.IsOfflineError(err) {
// It is fine to just ignore this, the next command will retry the API and eventually we will respond to any pending dump requests
dumpRequests = []*shared.DumpRequest{}
hctx.GetLogger().Infof("Failed to check for dump requests because we failed to connect to the remote server!")
} else {
lib.CheckFatalError(err)
}
}
if len(dumpRequests) > 0 {
lib.CheckFatalError(lib.RetrieveAdditionalEntriesFromRemote(ctx))
entries, err := lib.Search(ctx, db, "", 0)
lib.CheckFatalError(err)
var encEntries []*shared.EncHistoryEntry
for _, entry := range entries {
enc, err := data.EncryptHistoryEntry(config.UserSecret, *entry)
lib.CheckFatalError(err)
encEntries = append(encEntries, &enc)
}
reqBody, err := json.Marshal(encEntries)
lib.CheckFatalError(err)
for _, dumpRequest := range dumpRequests {
if !config.IsOffline {
_, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId+"&source_device_id="+config.DeviceId, "application/json", reqBody)
lib.CheckFatalError(err)
}
}
}
}
// Handle deletion requests
if shouldCheckForDeletionRequests {
lib.CheckFatalError(lib.ProcessDeletionRequests(ctx))
}
if config.BetaMode {
db.Commit()
}
}
func deletePresavedEntries(ctx context.Context, entry *data.HistoryEntry) error {
db := hctx.GetDb(ctx)
// Create the query to find the presaved entries
query := "cwd:" + entry.CurrentWorkingDirectory
query += " start_time:" + strconv.FormatInt(entry.StartTime.Unix(), 10)
query += " end_time:1970/01/01_00:00:00_+00:00"
matchingEntryQuery, err := lib.MakeWhereQueryFromSearch(ctx, db, query)
if err != nil {
return fmt.Errorf("failed to query for pre-saved history entry: %w", err)
}
matchingEntryQuery = matchingEntryQuery.Where("command = ?", entry.Command).Session(&gorm.Session{})
// Get the presaved entry since we need it for doing remote deletes
var presavedEntry data.HistoryEntry
res := matchingEntryQuery.Find(&presavedEntry)
if res.Error != nil {
return fmt.Errorf("failed to search for presaved entry for cmd=%#v: %w", entry.Command, res.Error)
}
// Delete presaved entries locally
deletePresavedEntryFunc := func() error {
res := matchingEntryQuery.Delete(&data.HistoryEntry{})
if res.Error != nil {
return fmt.Errorf("failed to delete pre-saved history entry (expected command=%#v): %w", entry.Command, res.Error)
}
if res.RowsAffected > 1 {
return fmt.Errorf("attempted to delete pre-saved entry, but something went wrong since we deleted %d rows", res.RowsAffected)
}
return nil
}
err = lib.RetryingDbFunction(deletePresavedEntryFunc)
if err != nil {
return err
}
// And delete it remotely
config := hctx.GetConf(ctx)
var deletionRequest shared.DeletionRequest
deletionRequest.SendTime = time.Now()
deletionRequest.UserId = data.UserId(config.UserSecret)
deletionRequest.Messages.Ids = append(deletionRequest.Messages.Ids,
// Note that we aren't specifying an EndTime here since pre-saved entries don't have an EndTime
shared.MessageIdentifier{DeviceId: presavedEntry.DeviceId, EntryId: presavedEntry.EntryId},
)
return lib.SendDeletionRequest(deletionRequest)
}
func init() {
rootCmd.AddCommand(saveHistoryEntryCmd)
rootCmd.AddCommand(presaveHistoryEntryCmd)
}
func handleDumpRequests(ctx context.Context, dumpRequests []*shared.DumpRequest) error {
db := hctx.GetDb(ctx)
config := hctx.GetConf(ctx)
if len(dumpRequests) > 0 {
lib.CheckFatalError(lib.RetrieveAdditionalEntriesFromRemote(ctx))
entries, err := lib.Search(ctx, db, "", 0)
lib.CheckFatalError(err)
var encEntries []*shared.EncHistoryEntry
for _, entry := range entries {
enc, err := data.EncryptHistoryEntry(config.UserSecret, *entry)
lib.CheckFatalError(err)
encEntries = append(encEntries, &enc)
}
reqBody, err := json.Marshal(encEntries)
lib.CheckFatalError(err)
for _, dumpRequest := range dumpRequests {
if !config.IsOffline {
_, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId+"&source_device_id="+config.DeviceId, "application/json", reqBody)
lib.CheckFatalError(err)
}
}
}
return nil
}
func buildPreArgsHistoryEntry(ctx context.Context) (*data.HistoryEntry, error) {
var entry data.HistoryEntry
@ -274,6 +299,9 @@ func buildPreArgsHistoryEntry(ctx context.Context) (*data.HistoryEntry, error) {
config := hctx.GetConf(ctx)
entry.DeviceId = config.DeviceId
// entry ID
entry.EntryId = uuid.Must(uuid.NewRandom()).String()
// custom columns
cc, err := buildCustomColumns(ctx)
if err != nil {

11
client/cmd/status.go
View File

@ -22,7 +22,6 @@ var statusCmd = &cobra.Command{
if *verbose {
fmt.Printf("User ID: %s\n", data.UserId(config.UserSecret))
fmt.Printf("Device ID: %s\n", config.DeviceId)
printDumpStatus(config)
printOnlineStatus(config)
}
fmt.Printf("Commit Hash: %s\n", lib.GitCommit)
@ -42,16 +41,6 @@ func printOnlineStatus(config hctx.ClientConfig) {
}
}
func printDumpStatus(config hctx.ClientConfig) {
dumpRequests, err := lib.GetDumpRequests(config)
lib.CheckFatalError(err)
fmt.Printf("Dump Requests: ")
for _, d := range dumpRequests {
fmt.Printf("%#v, ", *d)
}
fmt.Print("\n")
}
func init() {
rootCmd.AddCommand(statusCmd)
verbose = statusCmd.Flags().BoolP("verbose", "v", false, "Display verbose hiSHtory information")

4
client/data/data.go
View File

@ -15,7 +15,6 @@ import (
"time"
"github.com/ddworken/hishtory/shared"
"github.com/google/uuid"
)
const (
@ -39,6 +38,7 @@ type HistoryEntry struct {
StartTime time.Time `json:"start_time" gorm:"uniqueIndex:compositeindex"`
EndTime time.Time `json:"end_time" gorm:"uniqueIndex:compositeindex,index:end_time_index"`
DeviceId string `json:"device_id" gorm:"uniqueIndex:compositeindex"`
EntryId string `json:"entry_id" gorm:"uniqueIndex:compositeindex,uniqueIndex:entry_id_index"`
CustomColumns CustomColumns `json:"custom_columns"`
}
@ -136,7 +136,7 @@ func EncryptHistoryEntry(userSecret string, entry HistoryEntry) (shared.EncHisto
Nonce: nonce,
UserId: UserId(userSecret),
Date: entry.EndTime,
EncryptedId: uuid.Must(uuid.NewRandom()).String(),
EncryptedId: entry.EntryId,
ReadCount: 0,
}, nil
}

1
client/hctx/hctx.go
View File

@ -103,6 +103,7 @@ func OpenLocalSqliteDb() (*gorm.DB, error) {
db.AutoMigrate(&data.HistoryEntry{})
db.Exec("PRAGMA journal_mode = WAL")
db.Exec("CREATE INDEX IF NOT EXISTS end_time_index ON history_entries(end_time)")
db.Exec("CREATE INDEX IF NOT EXISTS entry_id_index ON history_entries(entry_id)")
return db, nil
}

26
client/lib/lib.go
View File

@ -303,6 +303,7 @@ func ImportHistory(ctx context.Context, shouldReadStdin, force bool) (int, error
StartTime: time.Now().UTC(),
EndTime: time.Now().UTC(),
DeviceId: config.DeviceId,
EntryId: uuid.Must(uuid.NewRandom()).String(),
}
err = ReliableDbCreate(db, entry)
if err != nil {
@ -597,10 +598,17 @@ func ProcessDeletionRequests(ctx context.Context) error {
if err != nil {
return err
}
return HandleDeletionRequests(ctx, deletionRequests)
}
func HandleDeletionRequests(ctx context.Context, deletionRequests []*shared.DeletionRequest) error {
db := hctx.GetDb(ctx)
for _, request := range deletionRequests {
for _, entry := range request.Messages.Ids {
res := db.Where("device_id = ? AND end_time = ?", entry.DeviceId, entry.Date).Delete(&data.HistoryEntry{})
// Note that entry.EndTime is not always present (for pre-saved entries). And likewise,
// entry.EntryId is not always present for older entries. So we just check that one of them matches.
tx := db.Where("device_id = ? AND (end_time = ? OR entry_id = ?)", entry.DeviceId, entry.EndTime, entry.EntryId)
res := tx.Delete(&data.HistoryEntry{})
if res.Error != nil {
return fmt.Errorf("DB error: %w", res.Error)
}
@ -872,22 +880,6 @@ func unescape(query string) string {
return string(newQuery)
}
func GetDumpRequests(config hctx.ClientConfig) ([]*shared.DumpRequest, error) {
if config.IsOffline {
return make([]*shared.DumpRequest, 0), nil
}
resp, err := ApiGet("/api/v1/get-dump-requests?user_id=" + data.UserId(config.UserSecret) + "&device_id=" + config.DeviceId)
if IsOfflineError(err) {
return []*shared.DumpRequest{}, nil
}
if err != nil {
return nil, err
}
var dumpRequests []*shared.DumpRequest
err = json.Unmarshal(resp, &dumpRequests)
return dumpRequests, err
}
func SendDeletionRequest(deletionRequest shared.DeletionRequest) error {
data, err := json.Marshal(deletionRequest)
if err != nil {

4
client/tui/tui.go
View File

@ -601,7 +601,9 @@ func deleteHistoryEntry(ctx context.Context, entry data.HistoryEntry) error {
UserId: data.UserId(hctx.GetConf(ctx).UserSecret),
SendTime: time.Now(),
}
dr.Messages.Ids = append(dr.Messages.Ids, shared.MessageIdentifier{Date: entry.EndTime, DeviceId: entry.DeviceId})
dr.Messages.Ids = append(dr.Messages.Ids,
shared.MessageIdentifier{DeviceId: entry.DeviceId, EndTime: entry.EndTime, EntryId: entry.EntryId},
)
return lib.SendDeletionRequest(dr)
}

42
shared/data.go
View File

@ -9,22 +9,17 @@ import (
// Represents an encrypted history entry
type EncHistoryEntry struct {
EncryptedData []byte `json:"enc_data"`
Nonce []byte `json:"nonce"`
DeviceId string `json:"device_id"`
UserId string `json:"user_id"`
Date time.Time `json:"time"`
EncryptedId string `json:"id"`
ReadCount int `json:"read_count"`
EncryptedData []byte `json:"enc_data"`
Nonce []byte `json:"nonce"`
DeviceId string `json:"device_id"`
UserId string `json:"user_id"`
// Note that EncHistoryEntry.Date == HistoryEntry.EndTime
Date time.Time `json:"time"`
// Note that EncHistoryEntry.EncryptedId == HistoryEntry.Id (for entries created after pre-saving support)
EncryptedId string `json:"encrypted_id"`
ReadCount int `json:"read_count"`
}
/*
Manually created the indices:
CREATE INDEX CONCURRENTLY device_id_idx ON enc_history_entries USING btree(device_id);
CREATE INDEX CONCURRENTLY read_count_idx ON enc_history_entries USING btree(read_count);
CREATE INDEX CONCURRENTLY redact_idx ON enc_history_entries USING btree(user_id, device_id, date);
*/
type Device struct {
UserId string `json:"user_id"`
DeviceId string `json:"device_id"`
@ -81,14 +76,19 @@ type MessageIdentifiers struct {
Ids []MessageIdentifier `json:"message_ids"`
}
// Identifies a single history entry based on the device that recorded the entry, and the end time. Note that
// Identifies a single history entry based on the device that recorded the entry, and additional metadata. Note that
// this does not include the command itself since that would risk including the sensitive data that is meant
// to be deleted
type MessageIdentifier struct {
// The device that the entry was recorded on (NOT the device where it is stored/requesting deletion)
DeviceId string `json:"device_id"`
// The timestamp when the message finished running
Date time.Time `json:"date"`
// The timestamp when the command finished running. Serialized as "date" for legacy compatibility.
EndTime time.Time `json:"date"`
// The entry ID of the command.
// Note this field was added as part of supporting pre-saving commands, so older clients do not set this field
// And even for new clients, it may contain a per-device entry ID. For pre-saved entries, this is guaranteed to
// be present.
EntryId string `json:"entry_id"`
}
func (m *MessageIdentifiers) Scan(value interface{}) error {
@ -114,11 +114,11 @@ type Feedback struct {
Feedback string `json:"feedback"`
}
// Response from submitting new history entries. Contains metadata that is used to avoid making additional round-trip
// requests to the hishtory backend.
// Response from submitting new history entries. Contains deletion requests and dump requests to avoid
// extra round-trip requests to the hishtory backend.
type SubmitResponse struct {
HaveDumpRequests bool `json:"have_dump_requests"`
HaveDeletionRequests bool `json:"have_deletion_requests"`
DumpRequests []*DumpRequest `json:"dump_requests"`
DeletionRequests []*DeletionRequest `json:"deletion_requests"`
}
func Chunks[k any](slice []k, chunkSize int) [][]k {

2
shared/testutils/testutils.go
View File

@ -17,6 +17,7 @@ import (
"github.com/ddworken/hishtory/client/data"
"github.com/google/go-cmp/cmp"
"github.com/google/uuid"
)
const (
@ -326,6 +327,7 @@ func MakeFakeHistoryEntry(command string) data.HistoryEntry {
StartTime: time.Unix(fakeHistoryTimestamp, 0).UTC(),
EndTime: time.Unix(fakeHistoryTimestamp+3, 0).UTC(),
DeviceId: "fake_device_id",
EntryId: uuid.Must(uuid.NewRandom()).String(),
}
}