Implement pre-saving feature to ensure that long-running/non-terminating commands are saved in hishtory

client/cmd/configGet.go

@@ -40,6 +40,16 @@ var getFilterDuplicateCommandsCmd = &cobra.Command{
 	},
 }
 
+var getBetaModeCmd = &cobra.Command{
+	Use:   "beta-mode",
+	Short: "Enable beta-mode to opt-in to unreleased features",
+	Run: func(cmd *cobra.Command, args []string) {
+		ctx := hctx.MakeContext()
+		config := hctx.GetConf(ctx)
+		fmt.Println(config.BetaMode)
+	},
+}
+
 var getDisplayedColumnsCmd = &cobra.Command{
 	Use:   "displayed-columns",
 	Short: "The list of columns that hishtory displays",
@@ -87,4 +97,5 @@ func init() {
 	configGetCmd.AddCommand(getDisplayedColumnsCmd)
 	configGetCmd.AddCommand(getTimestampFormatCmd)
 	configGetCmd.AddCommand(getCustomColumnsCmd)
+	configGetCmd.AddCommand(getBetaModeCmd)
 }

client/cmd/configSet.go

@@ -53,6 +53,23 @@ var setFilterDuplicateCommandsCmd = &cobra.Command{
 	},
 }
 
+var setBetaModeCommand = &cobra.Command{
+	Use:       "beta-mode",
+	Short:     "Enable beta-mode to opt-in to unreleased features",
+	Args:      cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
+	ValidArgs: []string{"true", "false"},
+	Run: func(cmd *cobra.Command, args []string) {
+		val := args[0]
+		if val != "true" && val != "false" {
+			log.Fatalf("Unexpected config value %s, must be one of: true, false", val)
+		}
+		ctx := hctx.MakeContext()
+		config := hctx.GetConf(ctx)
+		config.BetaMode = (val == "true")
+		lib.CheckFatalError(hctx.SetConfig(config))
+	},
+}
+
 var setDisplayedColumnsCmd = &cobra.Command{
 	Use:   "displayed-columns",
 	Short: "The list of columns that hishtory displays",
@@ -83,4 +100,5 @@ func init() {
 	configSetCmd.AddCommand(setFilterDuplicateCommandsCmd)
 	configSetCmd.AddCommand(setDisplayedColumnsCmd)
 	configSetCmd.AddCommand(setTimestampFormatCmd)
+	configSetCmd.AddCommand(setBetaModeCommand)
 }

client/cmd/saveHistoryEntry.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/ddworken/hishtory/client/data"
@@ -26,6 +28,17 @@ var saveHistoryEntryCmd = &cobra.Command{
 	},
 }
 
+var presaveHistoryEntryCmd = &cobra.Command{
+	Use:                "presaveHistoryEntry",
+	Hidden:             true,
+	Short:              "[Internal-only] The command used to pre-save history entries that haven't yet finished running",
+	DisableFlagParsing: true,
+	Run: func(cmd *cobra.Command, args []string) {
+		ctx := hctx.MakeContext()
+		presaveHistoryEntry(ctx)
+	},
+}
+
 func maybeUploadSkippedHistoryEntries(ctx *context.Context) error {
 	config := hctx.GetConf(ctx)
 	if !config.HaveMissedUploads {
@@ -63,6 +76,48 @@ func maybeUploadSkippedHistoryEntries(ctx *context.Context) error {
 	return nil
 }
 
+func presaveHistoryEntry(ctx *context.Context) {
+	config := hctx.GetConf(ctx)
+	if !config.IsEnabled {
+		return
+	}
+	if !config.BetaMode {
+		return
+	}
+
+	// Build the basic entry with metadata retrieved from runtime
+	entry, err := lib.BuildPreArgsHistoryEntry(ctx)
+	lib.CheckFatalError(err)
+	if entry == nil {
+		return
+	}
+
+	// Augment it with os.Args
+	entry.Command = lib.TrimTrailingWhitespace(os.Args[3])
+	if strings.HasPrefix(" ", entry.Command) {
+		// Don't save commands that start with a space
+		return
+	}
+	fmt.Println(entry.Command)
+	startTime, err := lib.ParseCrossPlatformInt(os.Args[4])
+	lib.CheckFatalError(err)
+	entry.StartTime = time.Unix(startTime, 0)
+	entry.EndTime = time.Unix(0, 0)
+
+	// And persist it locally.
+	db := hctx.GetDb(ctx)
+	err = lib.ReliableDbCreate(db, *entry)
+	lib.CheckFatalError(err)
+	db.Commit()
+
+	// Note that we aren't persisting these half-entries remotely,
+	// since they should be updated with the rest of the information very soon. If they
+	// are never updated (e.g. due to a long-running command that never finishes), then
+	// they will only be available on this device. That isn't perfect since it means
+	// history entries can get out of sync, but it is probably good enough.
+	// TODO: Consider improving this
+}
+
 func saveHistoryEntry(ctx *context.Context) {
 	config := hctx.GetConf(ctx)
 	if !config.IsEnabled {
@@ -75,9 +130,24 @@ func saveHistoryEntry(ctx *context.Context) {
 		hctx.GetLogger().Infof("Skipping saving a history entry because we did not build a history entry (was the command prefixed with a space and/or empty?)\n")
 		return
 	}
+	db := hctx.GetDb(ctx)
+
+	// Drop any entries from pre-saving since they're no longer needed
+	if config.BetaMode {
+		tx, err := lib.MakeWhereQueryFromSearch(ctx, db, "cwd:"+entry.CurrentWorkingDirectory+" start_time:"+strconv.FormatInt(entry.StartTime.Unix(), 10))
+		if err != nil {
+			lib.CheckFatalError(fmt.Errorf("failed to query for pre-saved history entries: %s", err))
+		}
+		res := tx.Delete(&data.HistoryEntry{})
+		if res.Error != nil {
+			lib.CheckFatalError(fmt.Errorf("failed to delete pre-saved history entries: %s", res.Error))
+		}
+		if res.RowsAffected > 1 {
+			lib.CheckFatalError(fmt.Errorf("attempted to delete pre-saved entry, but something went wrong since we deleted %d rows", res.RowsAffected))
+		}
+	}
 
 	// Persist it locally
-	db := hctx.GetDb(ctx)
 	err = lib.ReliableDbCreate(db, *entry)
 	lib.CheckFatalError(err)
 
@@ -133,8 +203,13 @@ func saveHistoryEntry(ctx *context.Context) {
 
 	// Handle deletion requests
 	lib.CheckFatalError(lib.ProcessDeletionRequests(ctx))
+
+	if config.BetaMode {
+		db.Commit()
+	}
 }
 
 func init() {
 	rootCmd.AddCommand(saveHistoryEntryCmd)
+	rootCmd.AddCommand(presaveHistoryEntryCmd)
 }

client/hctx/hctx.go

@@ -178,6 +178,9 @@ type ClientConfig struct {
 	FilterDuplicateCommands bool `json:"filter_duplicate_commands"`
 	// A format string for the timestamp
 	TimestampFormat string `json:"timestamp_format"`
+	// Beta mode, enables unspecified additional beta features
+	// Currently: This enables pre-saving of history entries to better handle long-running commands
+	BetaMode bool `json:"beta_mode"`
 }
 
 type CustomColumnDefinition struct {

client/lib/config.fish

@@ -2,6 +2,7 @@ function _hishtory_post_exec --on-event fish_postexec
     # Runs after <ENTER>, but before the command is executed
     set --global _hishtory_command $argv
     set --global _hishtory_start_time (date +%s)
+    # TODO: Implement pre-saving for fish
 end
 
 set --global _hishtory_first_prompt 1

client/lib/config.sh

@@ -14,6 +14,9 @@ function __hishtory_precommand() {
 
   # Run before every command
   HISHTORY_START_TIME=`date +%s`
+  if ! [ -z "BASH_COMMAND " ] && [ "$BASH_COMMAND" != "__hishtory_postcommand" ]; then
+    hishtory presaveHistoryEntry bash "$BASH_COMMAND" $HISHTORY_START_TIME
+  fi
 }
 trap "__hishtory_precommand" DEBUG
 

client/lib/config.zsh

@@ -9,6 +9,9 @@ function _hishtory_add() {
     # $1 contains the command that was run 
     _hishtory_command=$1
     _hishtory_start_time=`date +%s`
+    if ! [ -z "$_hishtory_command " ]; then
+        hishtory presaveHistoryEntry zsh "$_hishtory_command" $_hishtory_start_time
+    fi
 }
 
 function _hishtory_precmd() {

client/lib/lib.go

@@ -83,22 +83,9 @@ func getCwdWithoutSubstitution() (string, error) {
 	return "", err
 }
 
-func BuildHistoryEntry(ctx *context.Context, args []string) (*data.HistoryEntry, error) {
+func BuildPreArgsHistoryEntry(ctx *context.Context) (*data.HistoryEntry, error) {
-	if len(args) < 6 {
-		hctx.GetLogger().Warnf("BuildHistoryEntry called with args=%#v, which has too few entries! This can happen in specific edge cases for newly opened terminals and is likely not a problem.", args)
-		return nil, nil
-	}
-	shell := args[2]
-
 	var entry data.HistoryEntry
 
-	// exitCode
-	exitCode, err := strconv.Atoi(args[3])
-	if err != nil {
-		return nil, fmt.Errorf("failed to build history entry: %v", err)
-	}
-	entry.ExitCode = exitCode
-
 	// user
 	user, err := user.Current()
 	if err != nil {
@@ -114,8 +101,48 @@ func BuildHistoryEntry(ctx *context.Context, args []string) (*data.HistoryEntry,
 	entry.CurrentWorkingDirectory = cwd
 	entry.HomeDirectory = homedir
 
+	// hostname
+	hostname, err := os.Hostname()
+	if err != nil {
+		return nil, fmt.Errorf("failed to build history entry: %v", err)
+	}
+	entry.Hostname = hostname
+
+	// device ID
+	config := hctx.GetConf(ctx)
+	entry.DeviceId = config.DeviceId
+
+	// custom columns
+	cc, err := buildCustomColumns(ctx)
+	if err != nil {
+		return nil, err
+	}
+	entry.CustomColumns = cc
+
+	return &entry, nil
+}
+
+func BuildHistoryEntry(ctx *context.Context, args []string) (*data.HistoryEntry, error) {
+	if len(args) < 6 {
+		hctx.GetLogger().Warnf("BuildHistoryEntry called with args=%#v, which has too few entries! This can happen in specific edge cases for newly opened terminals and is likely not a problem.", args)
+		return nil, nil
+	}
+	shell := args[2]
+
+	entry, err := BuildPreArgsHistoryEntry(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// exitCode
+	exitCode, err := strconv.Atoi(args[3])
+	if err != nil {
+		return nil, fmt.Errorf("failed to build history entry: %v", err)
+	}
+	entry.ExitCode = exitCode
+
 	// start time
-	seconds, err := parseCrossPlatformInt(args[5])
+	seconds, err := ParseCrossPlatformInt(args[5])
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse start time %s as int: %v", args[5], err)
 	}
@@ -144,7 +171,7 @@ func BuildHistoryEntry(ctx *context.Context, args []string) (*data.HistoryEntry,
 		}
 		entry.Command = cmd
 	} else if shell == "zsh" || shell == "fish" {
-		cmd := strings.TrimSuffix(strings.TrimSuffix(args[4], "\n"), " ")
+		cmd := TrimTrailingWhitespace(args[4])
 		if strings.HasPrefix(cmd, " ") {
 			// Don't save commands that start with a space
 			return nil, nil
@@ -158,25 +185,11 @@ func BuildHistoryEntry(ctx *context.Context, args []string) (*data.HistoryEntry,
 		return nil, nil
 	}
 
-	// hostname
+	return entry, nil
-	hostname, err := os.Hostname()
+}
-	if err != nil {
-		return nil, fmt.Errorf("failed to build history entry: %v", err)
-	}
-	entry.Hostname = hostname
 
-	// device ID
+func TrimTrailingWhitespace(s string) string {
-	config := hctx.GetConf(ctx)
+	return strings.TrimSuffix(strings.TrimSuffix(s, "\n"), " ")
-	entry.DeviceId = config.DeviceId
-
-	// custom columns
-	cc, err := buildCustomColumns(ctx)
-	if err != nil {
-		return nil, err
-	}
-	entry.CustomColumns = cc
-
-	return &entry, nil
 }
 
 func buildCustomColumns(ctx *context.Context) (data.CustomColumns, error) {
@@ -319,7 +332,7 @@ func maybeSkipBashHistTimePrefix(cmdLine string) (string, error) {
 	return re.ReplaceAllLiteralString(cmdLine, ""), nil
 }
 
-func parseCrossPlatformInt(data string) (int64, error) {
+func ParseCrossPlatformInt(data string) (int64, error) {
 	data = strings.TrimSuffix(data, "N")
 	return strconv.ParseInt(data, 10, 64)
 }
@@ -446,7 +459,12 @@ func buildTableRow(ctx *context.Context, columnNames []string, entry data.Histor
 		case "Timestamp":
 			row = append(row, entry.StartTime.Format(hctx.GetConf(ctx).TimestampFormat))
 		case "Runtime":
-			row = append(row, entry.EndTime.Sub(entry.StartTime).Round(time.Millisecond).String())
+			if entry.EndTime == time.Unix(0, 0) {
+				// An EndTime of zero means this is a pre-saved entry that never finished
+				row = append(row, "N/A")
+			} else {
+				row = append(row, entry.EndTime.Sub(entry.StartTime).Round(time.Millisecond).String())
+			}
 		case "Exit Code":
 			row = append(row, fmt.Sprintf("%d", entry.ExitCode))
 		case "Command":
@@ -977,6 +995,9 @@ func ReliableDbCreate(db *gorm.DB, entry interface{}) error {
 	for i = 0; i < 10; i++ {
 		result := db.Create(entry)
 		err = result.Error
+		if err == nil {
+			return nil
+		}
 		if err != nil {
 			errMsg := err.Error()
 			if errMsg == "database is locked (5) (SQLITE_BUSY)" || errMsg == "database is locked (261)" {
@@ -1161,7 +1182,7 @@ func Search(ctx *context.Context, db *gorm.DB, query string, limit int) ([]*data
 	if err != nil {
 		return nil, err
 	}
-	tx = tx.Order("end_time DESC")
+	tx = tx.Order("start_time DESC")
 	if limit > 0 {
 		tx = tx.Limit(limit)
 	}
@@ -1205,6 +1226,16 @@ func parseAtomizedToken(ctx *context.Context, token string) (string, interface{}
 			return "", nil, nil, fmt.Errorf("failed to parse after:%s as a timestamp: %v", val, err)
 		}
 		return "(CAST(strftime(\"%s\",start_time) AS INTEGER) > ?)", t.Unix(), nil, nil
+	case "start_time":
+		// Note that this atom probably isn't useful for interactive usage since it does exact matching, but we use it
+		// internally for pre-saving history entries.
+		t, err := parseTimeGenerously(val)
+		if err != nil {
+			return "", nil, nil, fmt.Errorf("failed to parse after:%s as a timestamp: %v", val, err)
+		}
+		return "(CAST(strftime(\"%s\",start_time) AS INTEGER) = ?)", t.Unix(), nil, nil
+	case "command":
+		return "(instr(command, ?) > 0)", val, nil, nil
 	default:
 		knownCustomColumns := make([]string, 0)
 		// Get custom columns that are defined on this machine

client/lib/lib_test.go

@@ -318,12 +318,12 @@ func TestAddToDbIfNew(t *testing.T) {
 }
 
 func TestParseCrossPlatformInt(t *testing.T) {
-	res, err := parseCrossPlatformInt("123")
+	res, err := ParseCrossPlatformInt("123")
 	testutils.Check(t, err)
 	if res != 123 {
 		t.Fatalf("failed to parse cross platform int %d", res)
 	}
-	res, err = parseCrossPlatformInt("123N")
+	res, err = ParseCrossPlatformInt("123N")
 	testutils.Check(t, err)
 	if res != 123 {
 		t.Fatalf("failed to parse cross platform int %d", res)
@@ -493,6 +493,11 @@ func TestParseTimeGenerously(t *testing.T) {
 	if ts.Year() != 2006 || ts.Month() != time.January || ts.Day() != 2 || ts.Hour() != 0 || ts.Minute() != 0 || ts.Second() != 0 {
 		t.Fatalf("parsed time incorrectly: %d", ts.Unix())
 	}
+	ts, err = parseTimeGenerously("1693163976")
+	testutils.Check(t, err)
+	if ts.Year() != 2023 || ts.Month() != time.August || ts.Day() != 27 || ts.Hour() != 12 || ts.Minute() != 19 || ts.Second() != 36 {
+		t.Fatalf("parsed time incorrectly: %d %s", ts.Unix(), ts.GoString())
+	}
 }
 
 func TestUnescape(t *testing.T) {