Periodically call query endpoint and retrieve data to ensure that the local DB is always reasonably up to date

client/cmd/query.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"strings"
 
 	"github.com/ddworken/hishtory/client/hctx"
@@ -61,9 +62,34 @@ var exportCmd = &cobra.Command{
 	},
 }
 
+var updateLocalDbFromRemoteCmd = &cobra.Command{
+	Use:     "updateLocalDbFromRemote",
+	Hidden:  true,
+	Short:   "[Internal-only] Update local DB from remote",
+	GroupID: GROUP_ID_QUERYING,
+	Run: func(cmd *cobra.Command, args []string) {
+		// Periodically, run a query so as to ensure that the local DB stays mostly up to date and that we don't
+		// accumulate a large number of entries that this device doesn't know about. This ensures that queries
+		// are always reasonably complete and fast (even when offline).
+		ctx := hctx.MakeContext()
+		config := hctx.GetConf(ctx)
+		if config.IsOffline {
+			return
+		}
+		// TODO: Move this out of config.BetaMode only once we're confident in this change
+		if config.BetaMode {
+			// Do it a random percent of the time, which should be approximately often enough.
+			if rand.Intn(20) == 0 && !config.IsOffline {
+				lib.CheckFatalError(lib.RetrieveAdditionalEntriesFromRemote(ctx, "preload"))
+				lib.CheckFatalError(lib.ProcessDeletionRequests(ctx))
+			}
+		}
+	},
+}
+
 func export(ctx context.Context, query string) {
 	db := hctx.GetDb(ctx)
-	err := lib.RetrieveAdditionalEntriesFromRemote(ctx)
+	err := lib.RetrieveAdditionalEntriesFromRemote(ctx, "export")
 	if err != nil {
 		if lib.IsOfflineError(ctx, err) {
 			fmt.Println("Warning: hishtory is offline so this may be missing recent results from your other machines!")
@@ -80,7 +106,7 @@ func export(ctx context.Context, query string) {
 
 func query(ctx context.Context, query string) {
 	db := hctx.GetDb(ctx)
-	err := lib.RetrieveAdditionalEntriesFromRemote(ctx)
+	err := lib.RetrieveAdditionalEntriesFromRemote(ctx, "query")
 	if err != nil {
 		if lib.IsOfflineError(ctx, err) {
 			fmt.Println("Warning: hishtory is offline so this may be missing recent results from your other machines!")
@@ -113,4 +139,5 @@ func init() {
 	rootCmd.AddCommand(queryCmd)
 	rootCmd.AddCommand(tqueryCmd)
 	rootCmd.AddCommand(exportCmd)
+	rootCmd.AddCommand(updateLocalDbFromRemoteCmd)
 }

client/cmd/redact.go

@@ -26,7 +26,7 @@ var redactCmd = &cobra.Command{
 	DisableFlagParsing: true,
 	Run: func(cmd *cobra.Command, args []string) {
 		ctx := hctx.MakeContext()
-		lib.CheckFatalError(lib.RetrieveAdditionalEntriesFromRemote(ctx))
+		lib.CheckFatalError(lib.RetrieveAdditionalEntriesFromRemote(ctx, "redact"))
 		lib.CheckFatalError(lib.ProcessDeletionRequests(ctx))
 		query := strings.Join(args, " ")
 		lib.CheckFatalError(redact(ctx, query, os.Getenv("HISHTORY_REDACT_FORCE") != ""))

client/cmd/saveHistoryEntry.go

@@ -284,7 +284,7 @@ func handleDumpRequests(ctx context.Context, dumpRequests []*shared.DumpRequest)
 	db := hctx.GetDb(ctx)
 	config := hctx.GetConf(ctx)
 	if len(dumpRequests) > 0 {
-		lib.CheckFatalError(lib.RetrieveAdditionalEntriesFromRemote(ctx))
+		lib.CheckFatalError(lib.RetrieveAdditionalEntriesFromRemote(ctx, "newclient"))
 		entries, err := lib.Search(ctx, db, "", 0)
 		lib.CheckFatalError(err)
 		var encEntries []*shared.EncHistoryEntry

client/lib/config.fish

@@ -17,6 +17,7 @@ function __hishtory_on_prompt --on-event fish_prompt
     else if [ -n "$_hishtory_command" ]
         hishtory saveHistoryEntry fish $_hishtory_exit_code "$_hishtory_command" $_hishtory_start_time &  # Background Run
         # hishtory saveHistoryEntry fish $_hishtory_exit_code "$_hishtory_command" $_hishtory_start_time  # Foreground Run
+        hishtory updateLocalDbFromRemote &
         set --global -e _hishtory_command  # Unset _hishtory_command so we don't double-save entries when fish_prompt is invoked but fish_postexec isn't
     end 
 end

client/lib/config.sh

@@ -35,6 +35,8 @@ function __hishtory_postcommand() {
   # Run after every prompt
   (hishtory saveHistoryEntry bash $EXIT_CODE "`history 1`" $HISHTORY_START_TIME &) # Background Run
   # hishtory saveHistoryEntry bash $EXIT_CODE "`history 1`" $HISHTORY_START_TIME  # Foreground Run
+
+  (hishtory updateLocalDbFromRemote &)
 }
 PROMPT_COMMAND="__hishtory_postcommand; $PROMPT_COMMAND"
 export HISTTIMEFORMAT=$HISTTIMEFORMAT

client/lib/config.zsh

@@ -25,6 +25,7 @@ function _hishtory_precmd() {
     fi
     (hishtory saveHistoryEntry zsh $_hishtory_exit_code "$_hishtory_command" $_hishtory_start_time &)  # Background Run
     # hishtory saveHistoryEntry zsh $_hishtory_exit_code "$_hishtory_command" $_hishtory_start_time  # Foreground Run
+    (hishtory updateLocalDbFromRemote &)
 }
 
 _hishtory_widget() {

client/lib/lib.go

@@ -760,13 +760,13 @@ func Reupload(ctx context.Context) error {
 	})
 }
 
-func RetrieveAdditionalEntriesFromRemote(ctx context.Context) error {
+func RetrieveAdditionalEntriesFromRemote(ctx context.Context, queryReason string) error {
 	db := hctx.GetDb(ctx)
 	config := hctx.GetConf(ctx)
 	if config.IsOffline {
 		return nil
 	}
-	respBody, err := ApiGet(ctx, "/api/v1/query?device_id="+config.DeviceId+"&user_id="+data.UserId(config.UserSecret))
+	respBody, err := ApiGet(ctx, "/api/v1/query?device_id="+config.DeviceId+"&user_id="+data.UserId(config.UserSecret)+"&queryReason="+queryReason)
 	if IsOfflineError(ctx, err) {
 		return nil
 	}

client/tui/tui.go

@@ -696,7 +696,7 @@ func TuiQuery(ctx context.Context, initialQuery string) error {
 	}()
 	// Async: Retrieve additional entries from the backend
 	go func() {
-		err := lib.RetrieveAdditionalEntriesFromRemote(ctx)
+		err := lib.RetrieveAdditionalEntriesFromRemote(ctx, "tui")
 		if err != nil {
 			p.Send(err)
 		}