From 24900820884f02bd736b18c61e959d6ce4655530 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sun, 27 Aug 2023 22:05:24 -0700 Subject: [PATCH] no-op refactoring: Move history entry building code from lib.go to cmd file for saving history entries --- client/cmd/saveHistoryEntry.go | 304 ++++++++++++++++++++++++- client/cmd/saveHistoryEntry_test.go | 227 +++++++++++++++++++ client/lib/lib.go | 330 ++-------------------------- client/lib/lib_test.go | 218 ------------------ 4 files changed, 546 insertions(+), 533 deletions(-) create mode 100644 client/cmd/saveHistoryEntry_test.go diff --git a/client/cmd/saveHistoryEntry.go b/client/cmd/saveHistoryEntry.go index 9316e6f..ba2a479 100644 --- a/client/cmd/saveHistoryEntry.go +++ b/client/cmd/saveHistoryEntry.go @@ -1,12 +1,17 @@ package cmd import ( + "bytes" "context" "encoding/json" "fmt" "os" + "os/exec" + "os/user" + "regexp" "strconv" "strings" + "syscall" "time" "github.com/ddworken/hishtory/client/data" @@ -86,20 +91,20 @@ func presaveHistoryEntry(ctx *context.Context) { } // Build the basic entry with metadata retrieved from runtime - entry, err := lib.BuildPreArgsHistoryEntry(ctx) + entry, err := buildPreArgsHistoryEntry(ctx) lib.CheckFatalError(err) if entry == nil { return } // Augment it with os.Args - entry.Command = lib.TrimTrailingWhitespace(os.Args[3]) + entry.Command = trimTrailingWhitespace(os.Args[3]) if strings.HasPrefix(" ", entry.Command) { // Don't save commands that start with a space return } fmt.Println(entry.Command) - startTime, err := lib.ParseCrossPlatformInt(os.Args[4]) + startTime, err := parseCrossPlatformInt(os.Args[4]) lib.CheckFatalError(err) entry.StartTime = time.Unix(startTime, 0) entry.EndTime = time.Unix(0, 0) @@ -124,7 +129,7 @@ func saveHistoryEntry(ctx *context.Context) { hctx.GetLogger().Infof("Skipping saving a history entry because hishtory is disabled\n") return } - entry, err := lib.BuildHistoryEntry(ctx, os.Args) + entry, err := buildHistoryEntry(ctx, os.Args) lib.CheckFatalError(err) if entry == nil { hctx.GetLogger().Infof("Skipping saving a history entry because we did not build a history entry (was the command prefixed with a space and/or empty?)\n") @@ -213,3 +218,294 @@ func init() { rootCmd.AddCommand(saveHistoryEntryCmd) rootCmd.AddCommand(presaveHistoryEntryCmd) } + +func buildPreArgsHistoryEntry(ctx *context.Context) (*data.HistoryEntry, error) { + var entry data.HistoryEntry + + // user + user, err := user.Current() + if err != nil { + return nil, fmt.Errorf("failed to build history entry: %v", err) + } + entry.LocalUsername = user.Username + + // cwd and homedir + cwd, homedir, err := getCwd(ctx) + if err != nil { + return nil, fmt.Errorf("failed to build history entry: %v", err) + } + entry.CurrentWorkingDirectory = cwd + entry.HomeDirectory = homedir + + // hostname + hostname, err := os.Hostname() + if err != nil { + return nil, fmt.Errorf("failed to build history entry: %v", err) + } + entry.Hostname = hostname + + // device ID + config := hctx.GetConf(ctx) + entry.DeviceId = config.DeviceId + + // custom columns + cc, err := buildCustomColumns(ctx) + if err != nil { + return nil, err + } + entry.CustomColumns = cc + + return &entry, nil +} + +func buildHistoryEntry(ctx *context.Context, args []string) (*data.HistoryEntry, error) { + if len(args) < 6 { + hctx.GetLogger().Warnf("buildHistoryEntry called with args=%#v, which has too few entries! This can happen in specific edge cases for newly opened terminals and is likely not a problem.", args) + return nil, nil + } + shell := args[2] + + entry, err := buildPreArgsHistoryEntry(ctx) + if err != nil { + return nil, err + } + + // exitCode + exitCode, err := strconv.Atoi(args[3]) + if err != nil { + return nil, fmt.Errorf("failed to build history entry: %v", err) + } + entry.ExitCode = exitCode + + // start time + seconds, err := parseCrossPlatformInt(args[5]) + if err != nil { + return nil, fmt.Errorf("failed to parse start time %s as int: %v", args[5], err) + } + entry.StartTime = time.Unix(seconds, 0) + + // end time + entry.EndTime = time.Now() + + // command + if shell == "bash" { + cmd, err := getLastCommand(args[4]) + if err != nil { + return nil, fmt.Errorf("failed to build history entry: %v", err) + } + shouldBeSkipped, err := shouldSkipHiddenCommand(ctx, args[4]) + if err != nil { + return nil, fmt.Errorf("failed to check if command was hidden: %v", err) + } + if shouldBeSkipped || strings.HasPrefix(cmd, " ") { + // Don't save commands that start with a space + return nil, nil + } + cmd, err = maybeSkipBashHistTimePrefix(cmd) + if err != nil { + return nil, err + } + entry.Command = cmd + } else if shell == "zsh" || shell == "fish" { + cmd := trimTrailingWhitespace(args[4]) + if strings.HasPrefix(cmd, " ") { + // Don't save commands that start with a space + return nil, nil + } + entry.Command = cmd + } else { + return nil, fmt.Errorf("tried to save a hishtory entry from an unsupported shell=%#v", shell) + } + if strings.TrimSpace(entry.Command) == "" { + // Skip recording empty commands where the user just hits enter in their terminal + return nil, nil + } + + return entry, nil +} + +func trimTrailingWhitespace(s string) string { + return strings.TrimSuffix(strings.TrimSuffix(s, "\n"), " ") +} + +func buildCustomColumns(ctx *context.Context) (data.CustomColumns, error) { + ccs := data.CustomColumns{} + config := hctx.GetConf(ctx) + for _, cc := range config.CustomColumns { + cmd := exec.Command("bash", "-c", cc.ColumnCommand) + var stdout bytes.Buffer + cmd.Stdout = &stdout + var stderr bytes.Buffer + cmd.Stderr = &stderr + err := cmd.Start() + if err != nil { + return nil, fmt.Errorf("failed to execute custom command named %v (stdout=%#v, stderr=%#v)", cc.ColumnName, stdout.String(), stderr.String()) + } + err = cmd.Wait() + if err != nil { + // Log a warning, but don't crash. This way commands can exit with a different status and still work. + hctx.GetLogger().Warnf("failed to execute custom command named %v (stdout=%#v, stderr=%#v)", cc.ColumnName, stdout.String(), stderr.String()) + } + ccv := data.CustomColumn{ + Name: cc.ColumnName, + Val: strings.TrimSpace(stdout.String()), + } + ccs = append(ccs, ccv) + } + return ccs, nil +} + +func buildRegexFromTimeFormat(timeFormat string) string { + expectedRegex := "" + lastCharWasPercent := false + for _, char := range timeFormat { + if lastCharWasPercent { + if char == '%' { + expectedRegex += regexp.QuoteMeta(string(char)) + lastCharWasPercent = false + continue + } else if char == 't' { + expectedRegex += "\t" + } else if char == 'F' { + expectedRegex += buildRegexFromTimeFormat("%Y-%m-%d") + } else if char == 'Y' { + expectedRegex += "[0-9]{4}" + } else if char == 'G' { + expectedRegex += "[0-9]{4}" + } else if char == 'g' { + expectedRegex += "[0-9]{2}" + } else if char == 'C' { + expectedRegex += "[0-9]{2}" + } else if char == 'u' || char == 'w' { + expectedRegex += "[0-9]" + } else if char == 'm' { + expectedRegex += "[0-9]{2}" + } else if char == 'd' { + expectedRegex += "[0-9]{2}" + } else if char == 'D' { + expectedRegex += buildRegexFromTimeFormat("%m/%d/%y") + } else if char == 'T' { + expectedRegex += buildRegexFromTimeFormat("%H:%M:%S") + } else if char == 'H' || char == 'I' || char == 'U' || char == 'V' || char == 'W' || char == 'y' || char == 'Y' { + expectedRegex += "[0-9]{2}" + } else if char == 'M' { + expectedRegex += "[0-9]{2}" + } else if char == 'j' { + expectedRegex += "[0-9]{3}" + } else if char == 'S' || char == 'm' { + expectedRegex += "[0-9]{2}" + } else if char == 'c' { + // Note: Specific to the POSIX locale + expectedRegex += buildRegexFromTimeFormat("%a %b %e %H:%M:%S %Y") + } else if char == 'a' { + // Note: Specific to the POSIX locale + expectedRegex += "(Sun|Mon|Tue|Wed|Thu|Fri|Sat)" + } else if char == 'b' || char == 'h' { + // Note: Specific to the POSIX locale + expectedRegex += "(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)" + } else if char == 'e' || char == 'k' || char == 'l' { + expectedRegex += "[0-9 ]{2}" + } else if char == 'n' { + expectedRegex += "\n" + } else if char == 'p' { + expectedRegex += "(AM|PM)" + } else if char == 'P' { + expectedRegex += "(am|pm)" + } else if char == 's' { + expectedRegex += "\\d+" + } else if char == 'z' { + expectedRegex += "[+-][0-9]{4}" + } else if char == 'r' { + expectedRegex += buildRegexFromTimeFormat("%I:%M:%S %p") + } else if char == 'R' { + expectedRegex += buildRegexFromTimeFormat("%H:%M") + } else if char == 'x' { + expectedRegex += buildRegexFromTimeFormat("%m/%d/%y") + } else if char == 'X' { + expectedRegex += buildRegexFromTimeFormat("%H:%M:%S") + } else { + panic(fmt.Sprintf("buildRegexFromTimeFormat doesn't support %%%v, please open a bug against github.com/ddworken/hishtory", string(char))) + } + } else if char != '%' { + expectedRegex += regexp.QuoteMeta(string(char)) + } + lastCharWasPercent = false + if char == '%' { + lastCharWasPercent = true + } + } + return expectedRegex +} + +func maybeSkipBashHistTimePrefix(cmdLine string) (string, error) { + format := os.Getenv("HISTTIMEFORMAT") + if format == "" { + return cmdLine, nil + } + re, err := regexp.Compile("^" + buildRegexFromTimeFormat(format)) + if err != nil { + return "", fmt.Errorf("failed to parse regex for HISTTIMEFORMAT variable: %v", err) + } + return re.ReplaceAllLiteralString(cmdLine, ""), nil +} + +func parseCrossPlatformInt(data string) (int64, error) { + data = strings.TrimSuffix(data, "N") + return strconv.ParseInt(data, 10, 64) +} + +func getLastCommand(history string) (string, error) { + split := strings.SplitN(strings.TrimSpace(history), " ", 2) + if len(split) <= 1 { + return "", fmt.Errorf("got unexpected bash history line: %#v, please open a bug at github.com/ddworken/hishtory", history) + } + split = strings.SplitN(split[1], " ", 2) + if len(split) <= 1 { + return "", fmt.Errorf("got unexpected bash history line: %#v, please open a bug at github.com/ddworken/hishtory", history) + } + return split[1], nil +} + +func shouldSkipHiddenCommand(ctx *context.Context, historyLine string) (bool, error) { + config := hctx.GetConf(ctx) + if config.LastSavedHistoryLine == historyLine { + return true, nil + } + config.LastSavedHistoryLine = historyLine + err := hctx.SetConfig(config) + if err != nil { + return false, err + } + return false, nil +} + +func getCwd(ctx *context.Context) (string, string, error) { + cwd, err := getCwdWithoutSubstitution() + if err != nil { + return "", "", fmt.Errorf("failed to get cwd for last command: %v", err) + } + homedir := hctx.GetHome(ctx) + if cwd == homedir { + return "~/", homedir, nil + } + if strings.HasPrefix(cwd, homedir) { + return strings.Replace(cwd, homedir, "~", 1), homedir, nil + } + return cwd, homedir, nil +} + +func getCwdWithoutSubstitution() (string, error) { + cwd, err := os.Getwd() + if err == nil { + return cwd, nil + } + // Fall back to the syscall to see if that works, as an attempt to + // fix github.com/ddworken/hishtory/issues/69 + if syscall.ImplementsGetwd { + cwd, err = syscall.Getwd() + if err == nil { + return cwd, nil + } + } + return "", err +} diff --git a/client/cmd/saveHistoryEntry_test.go b/client/cmd/saveHistoryEntry_test.go new file mode 100644 index 0000000..cb76815 --- /dev/null +++ b/client/cmd/saveHistoryEntry_test.go @@ -0,0 +1,227 @@ +package cmd + +import ( + "os" + "os/user" + "strings" + "testing" + "time" + + "github.com/ddworken/hishtory/client/hctx" + "github.com/ddworken/hishtory/client/lib" + "github.com/ddworken/hishtory/shared/testutils" +) + +func TestBuildHistoryEntry(t *testing.T) { + defer testutils.BackupAndRestore(t)() + defer testutils.RunTestServer()() + testutils.Check(t, lib.Setup("", false)) + + // Test building an actual entry for bash + entry, err := buildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "bash", "120", " 123 ls /foo ", "1641774958"}) + testutils.Check(t, err) + if entry.ExitCode != 120 { + t.Fatalf("history entry has unexpected exit code: %v", entry.ExitCode) + } + user, err := user.Current() + if err != nil { + t.Fatalf("failed to retrieve user: %v", err) + } + if entry.LocalUsername != user.Username { + t.Fatalf("history entry has unexpected user name: %v", entry.LocalUsername) + } + if !strings.HasPrefix(entry.CurrentWorkingDirectory, "/") && !strings.HasPrefix(entry.CurrentWorkingDirectory, "~/") { + t.Fatalf("history entry has unexpected cwd: %v", entry.CurrentWorkingDirectory) + } + if !strings.HasPrefix(entry.HomeDirectory, "/") { + t.Fatalf("history entry has unexpected home directory: %v", entry.HomeDirectory) + } + if entry.Command != "ls /foo" { + t.Fatalf("history entry has unexpected command: %v", entry.Command) + } + if !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-09T") && !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-10T") { + t.Fatalf("history entry has incorrect date in the start time: %v", entry.StartTime.Format(time.RFC3339)) + } + if entry.StartTime.Unix() != 1641774958 { + t.Fatalf("history entry has incorrect Unix time in the start time: %v", entry.StartTime.Unix()) + } + + // Test building an entry for zsh + entry, err = buildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "zsh", "120", "ls /foo\n", "1641774958"}) + testutils.Check(t, err) + if entry.ExitCode != 120 { + t.Fatalf("history entry has unexpected exit code: %v", entry.ExitCode) + } + if entry.LocalUsername != user.Username { + t.Fatalf("history entry has unexpected user name: %v", entry.LocalUsername) + } + if !strings.HasPrefix(entry.CurrentWorkingDirectory, "/") && !strings.HasPrefix(entry.CurrentWorkingDirectory, "~/") { + t.Fatalf("history entry has unexpected cwd: %v", entry.CurrentWorkingDirectory) + } + if !strings.HasPrefix(entry.HomeDirectory, "/") { + t.Fatalf("history entry has unexpected home directory: %v", entry.HomeDirectory) + } + if entry.Command != "ls /foo" { + t.Fatalf("history entry has unexpected command: %v", entry.Command) + } + if !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-09T") && !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-10T") { + t.Fatalf("history entry has incorrect date in the start time: %v", entry.StartTime.Format(time.RFC3339)) + } + if entry.StartTime.Unix() != 1641774958 { + t.Fatalf("history entry has incorrect Unix time in the start time: %v", entry.StartTime.Unix()) + } + + // Test building an entry for fish + entry, err = buildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "fish", "120", "ls /foo\n", "1641774958"}) + testutils.Check(t, err) + if entry.ExitCode != 120 { + t.Fatalf("history entry has unexpected exit code: %v", entry.ExitCode) + } + if entry.LocalUsername != user.Username { + t.Fatalf("history entry has unexpected user name: %v", entry.LocalUsername) + } + if !strings.HasPrefix(entry.CurrentWorkingDirectory, "/") && !strings.HasPrefix(entry.CurrentWorkingDirectory, "~/") { + t.Fatalf("history entry has unexpected cwd: %v", entry.CurrentWorkingDirectory) + } + if !strings.HasPrefix(entry.HomeDirectory, "/") { + t.Fatalf("history entry has unexpected home directory: %v", entry.HomeDirectory) + } + if entry.Command != "ls /foo" { + t.Fatalf("history entry has unexpected command: %v", entry.Command) + } + if !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-09T") && !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-10T") { + t.Fatalf("history entry has incorrect date in the start time: %v", entry.StartTime.Format(time.RFC3339)) + } + if entry.StartTime.Unix() != 1641774958 { + t.Fatalf("history entry has incorrect Unix time in the start time: %v", entry.StartTime.Unix()) + } + + // Test building an entry that is empty, and thus not saved + entry, err = buildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "zsh", "120", " \n", "1641774958"}) + testutils.Check(t, err) + if entry != nil { + t.Fatalf("expected history entry to be nil") + } +} + +func TestBuildHistoryEntryWithTimestampStripping(t *testing.T) { + defer testutils.BackupAndRestoreEnv("HISTTIMEFORMAT")() + defer testutils.BackupAndRestore(t)() + defer testutils.RunTestServer()() + testutils.Check(t, lib.Setup("", false)) + + testcases := []struct { + input, histtimeformat, expectedCommand string + }{ + {" 123 ls /foo ", "", "ls /foo"}, + {" 2389 [2022-09-28 04:38:32 +0000] echo", "", "[2022-09-28 04:38:32 +0000] echo"}, + {" 2389 [2022-09-28 04:38:32 +0000] echo", "[%F %T %z] ", "echo"}, + } + for _, tc := range testcases { + conf := hctx.GetConf(hctx.MakeContext()) + conf.LastSavedHistoryLine = "" + testutils.Check(t, hctx.SetConfig(conf)) + + os.Setenv("HISTTIMEFORMAT", tc.histtimeformat) + entry, err := buildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "bash", "120", tc.input, "1641774958"}) + testutils.Check(t, err) + if entry == nil { + t.Fatalf("entry is unexpectedly nil") + } + if entry.Command != tc.expectedCommand { + t.Fatalf("buildHistoryEntry(%#v) returned %#v (expected=%#v)", tc.input, entry.Command, tc.expectedCommand) + } + } +} + +func TestParseCrossPlatformInt(t *testing.T) { + res, err := parseCrossPlatformInt("123") + testutils.Check(t, err) + if res != 123 { + t.Fatalf("failed to parse cross platform int %d", res) + } + res, err = parseCrossPlatformInt("123N") + testutils.Check(t, err) + if res != 123 { + t.Fatalf("failed to parse cross platform int %d", res) + } +} + +func TestBuildRegexFromTimeFormat(t *testing.T) { + testcases := []struct { + formatString, regex string + }{ + {"%Y ", "[0-9]{4} "}, + {"%F %T ", "[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} "}, + {"%F%T", "[0-9]{4}-[0-9]{2}-[0-9]{2}[0-9]{2}:[0-9]{2}:[0-9]{2}"}, + {"%%", "%"}, + {"%%%%", "%%"}, + {"%%%Y", "%[0-9]{4}"}, + {"%%%F%T", "%[0-9]{4}-[0-9]{2}-[0-9]{2}[0-9]{2}:[0-9]{2}:[0-9]{2}"}, + } + + for _, tc := range testcases { + if regex := buildRegexFromTimeFormat(tc.formatString); regex != tc.regex { + t.Fatalf("building a regex for %#v returned %#v (expected=%#v)", tc.formatString, regex, tc.regex) + } + } +} +func TestGetLastCommand(t *testing.T) { + testcases := []struct { + input, expectedOutput string + }{ + {" 0 ls", "ls"}, + {" 33 ls", "ls"}, + {" 33 ls --aaaa foo bar ", "ls --aaaa foo bar"}, + {" 2389 [2022-09-28 04:38:32 +0000] echo", "[2022-09-28 04:38:32 +0000] echo"}, + } + for _, tc := range testcases { + actualOutput, err := getLastCommand(tc.input) + testutils.Check(t, err) + if actualOutput != tc.expectedOutput { + t.Fatalf("getLastCommand(%#v) returned %#v (expected=%#v)", tc.input, actualOutput, tc.expectedOutput) + } + } +} + +func TestMaybeSkipBashHistTimePrefix(t *testing.T) { + defer testutils.BackupAndRestoreEnv("HISTTIMEFORMAT")() + + testcases := []struct { + env, cmdLine, expected string + }{ + {"%F %T ", "2019-07-12 13:02:31 sudo apt update", "sudo apt update"}, + {"%F %T ", "2019-07-12 13:02:31 ls a b", "ls a b"}, + {"%F %T ", "2019-07-12 13:02:31 ls a ", "ls a "}, + {"%F %T ", "2019-07-12 13:02:31 ls a", "ls a"}, + {"%F %T ", "2019-07-12 13:02:31 ls", "ls"}, + {"%F %T ", "2019-07-12 13:02:31 ls -Slah", "ls -Slah"}, + {"%F ", "2019-07-12 ls -Slah", "ls -Slah"}, + {"%F ", "2019-07-12 ls -Slah", "ls -Slah"}, + {"%Y", "2019ls -Slah", "ls -Slah"}, + {"%Y%Y", "20192020ls -Slah", "ls -Slah"}, + {"%Y%Y", "20192020ls -Slah20192020", "ls -Slah20192020"}, + {"", "ls -Slah", "ls -Slah"}, + {"[%F %T] ", "[2019-07-12 13:02:31] sudo apt update", "sudo apt update"}, + {"[%F a %T] ", "[2019-07-12 a 13:02:31] sudo apt update", "sudo apt update"}, + {"aaa ", "aaa sudo apt update", "sudo apt update"}, + {"%c ", "Sun Aug 19 02:56:02 2012 sudo apt update", "sudo apt update"}, + {"%c ", "Sun Aug 19 02:56:02 2012 ls", "ls"}, + {"[%c] ", "[Sun Aug 19 02:56:02 2012] ls", "ls"}, + {"[%c %t] ", "[Sun Aug 19 02:56:02 2012 ] ls", "ls"}, + {"[%c %t]", "[Sun Aug 19 02:56:02 2012 ]ls", "ls"}, + {"[%c %t]", "[Sun Aug 19 02:56:02 2012 ]foo", "foo"}, + {"[%c %t", "[Sun Aug 19 02:56:02 2012 foo", "foo"}, + {"[%F %T %z]", "[2022-09-28 04:17:06 +0000]foo", "foo"}, + {"[%F %T %z] ", "[2022-09-28 04:17:06 +0000] foo", "foo"}, + } + + for _, tc := range testcases { + os.Setenv("HISTTIMEFORMAT", tc.env) + stripped, err := maybeSkipBashHistTimePrefix(tc.cmdLine) + testutils.Check(t, err) + if stripped != tc.expected { + t.Fatalf("skipping the time prefix returned %#v (expected=%#v for %#v)", stripped, tc.expected, tc.cmdLine) + } + } +} diff --git a/client/lib/lib.go b/client/lib/lib.go index 746ab01..7f4f74b 100644 --- a/client/lib/lib.go +++ b/client/lib/lib.go @@ -18,7 +18,6 @@ import ( "path/filepath" "regexp" "runtime" - "strconv" "strings" "syscall" "time" @@ -52,316 +51,6 @@ var GitCommit string = "Unknown" // 256KB ought to be enough for any reasonable cmd var maxSupportedLineLengthForImport = 256_000 -func getCwd(ctx *context.Context) (string, string, error) { - cwd, err := getCwdWithoutSubstitution() - if err != nil { - return "", "", fmt.Errorf("failed to get cwd for last command: %v", err) - } - homedir := hctx.GetHome(ctx) - if cwd == homedir { - return "~/", homedir, nil - } - if strings.HasPrefix(cwd, homedir) { - return strings.Replace(cwd, homedir, "~", 1), homedir, nil - } - return cwd, homedir, nil -} - -func getCwdWithoutSubstitution() (string, error) { - cwd, err := os.Getwd() - if err == nil { - return cwd, nil - } - // Fall back to the syscall to see if that works, as an attempt to - // fix github.com/ddworken/hishtory/issues/69 - if syscall.ImplementsGetwd { - cwd, err = syscall.Getwd() - if err == nil { - return cwd, nil - } - } - return "", err -} - -func BuildPreArgsHistoryEntry(ctx *context.Context) (*data.HistoryEntry, error) { - var entry data.HistoryEntry - - // user - user, err := user.Current() - if err != nil { - return nil, fmt.Errorf("failed to build history entry: %v", err) - } - entry.LocalUsername = user.Username - - // cwd and homedir - cwd, homedir, err := getCwd(ctx) - if err != nil { - return nil, fmt.Errorf("failed to build history entry: %v", err) - } - entry.CurrentWorkingDirectory = cwd - entry.HomeDirectory = homedir - - // hostname - hostname, err := os.Hostname() - if err != nil { - return nil, fmt.Errorf("failed to build history entry: %v", err) - } - entry.Hostname = hostname - - // device ID - config := hctx.GetConf(ctx) - entry.DeviceId = config.DeviceId - - // custom columns - cc, err := buildCustomColumns(ctx) - if err != nil { - return nil, err - } - entry.CustomColumns = cc - - return &entry, nil -} - -func BuildHistoryEntry(ctx *context.Context, args []string) (*data.HistoryEntry, error) { - if len(args) < 6 { - hctx.GetLogger().Warnf("BuildHistoryEntry called with args=%#v, which has too few entries! This can happen in specific edge cases for newly opened terminals and is likely not a problem.", args) - return nil, nil - } - shell := args[2] - - entry, err := BuildPreArgsHistoryEntry(ctx) - if err != nil { - return nil, err - } - - // exitCode - exitCode, err := strconv.Atoi(args[3]) - if err != nil { - return nil, fmt.Errorf("failed to build history entry: %v", err) - } - entry.ExitCode = exitCode - - // start time - seconds, err := ParseCrossPlatformInt(args[5]) - if err != nil { - return nil, fmt.Errorf("failed to parse start time %s as int: %v", args[5], err) - } - entry.StartTime = time.Unix(seconds, 0) - - // end time - entry.EndTime = time.Now() - - // command - if shell == "bash" { - cmd, err := getLastCommand(args[4]) - if err != nil { - return nil, fmt.Errorf("failed to build history entry: %v", err) - } - shouldBeSkipped, err := shouldSkipHiddenCommand(ctx, args[4]) - if err != nil { - return nil, fmt.Errorf("failed to check if command was hidden: %v", err) - } - if shouldBeSkipped || strings.HasPrefix(cmd, " ") { - // Don't save commands that start with a space - return nil, nil - } - cmd, err = maybeSkipBashHistTimePrefix(cmd) - if err != nil { - return nil, err - } - entry.Command = cmd - } else if shell == "zsh" || shell == "fish" { - cmd := TrimTrailingWhitespace(args[4]) - if strings.HasPrefix(cmd, " ") { - // Don't save commands that start with a space - return nil, nil - } - entry.Command = cmd - } else { - return nil, fmt.Errorf("tried to save a hishtory entry from an unsupported shell=%#v", shell) - } - if strings.TrimSpace(entry.Command) == "" { - // Skip recording empty commands where the user just hits enter in their terminal - return nil, nil - } - - return entry, nil -} - -func TrimTrailingWhitespace(s string) string { - return strings.TrimSuffix(strings.TrimSuffix(s, "\n"), " ") -} - -func buildCustomColumns(ctx *context.Context) (data.CustomColumns, error) { - ccs := data.CustomColumns{} - config := hctx.GetConf(ctx) - for _, cc := range config.CustomColumns { - cmd := exec.Command("bash", "-c", cc.ColumnCommand) - var stdout bytes.Buffer - cmd.Stdout = &stdout - var stderr bytes.Buffer - cmd.Stderr = &stderr - err := cmd.Start() - if err != nil { - return nil, fmt.Errorf("failed to execute custom command named %v (stdout=%#v, stderr=%#v)", cc.ColumnName, stdout.String(), stderr.String()) - } - err = cmd.Wait() - if err != nil { - // Log a warning, but don't crash. This way commands can exit with a different status and still work. - hctx.GetLogger().Warnf("failed to execute custom command named %v (stdout=%#v, stderr=%#v)", cc.ColumnName, stdout.String(), stderr.String()) - } - ccv := data.CustomColumn{ - Name: cc.ColumnName, - Val: strings.TrimSpace(stdout.String()), - } - ccs = append(ccs, ccv) - } - return ccs, nil -} - -func stripZshWeirdness(cmd string) string { - // Zsh has this weird behavior where sometimes commands are saved in the hishtory file - // with a weird prefix. I've never been able to figure out why this happens, but we - // can at least strip it. - firstCommandBugRegex := regexp.MustCompile(`: \d+:\d;(.*)`) - matches := firstCommandBugRegex.FindStringSubmatch(cmd) - if len(matches) == 2 { - return matches[1] - } - return cmd -} - -func isBashWeirdness(cmd string) bool { - // Bash has this weird behavior where the it has entries like `#1664342754` in the - // history file. We want to skip these. - firstCommandBugRegex := regexp.MustCompile(`^#\d+\s+$`) - return firstCommandBugRegex.MatchString(cmd) -} - -func buildRegexFromTimeFormat(timeFormat string) string { - expectedRegex := "" - lastCharWasPercent := false - for _, char := range timeFormat { - if lastCharWasPercent { - if char == '%' { - expectedRegex += regexp.QuoteMeta(string(char)) - lastCharWasPercent = false - continue - } else if char == 't' { - expectedRegex += "\t" - } else if char == 'F' { - expectedRegex += buildRegexFromTimeFormat("%Y-%m-%d") - } else if char == 'Y' { - expectedRegex += "[0-9]{4}" - } else if char == 'G' { - expectedRegex += "[0-9]{4}" - } else if char == 'g' { - expectedRegex += "[0-9]{2}" - } else if char == 'C' { - expectedRegex += "[0-9]{2}" - } else if char == 'u' || char == 'w' { - expectedRegex += "[0-9]" - } else if char == 'm' { - expectedRegex += "[0-9]{2}" - } else if char == 'd' { - expectedRegex += "[0-9]{2}" - } else if char == 'D' { - expectedRegex += buildRegexFromTimeFormat("%m/%d/%y") - } else if char == 'T' { - expectedRegex += buildRegexFromTimeFormat("%H:%M:%S") - } else if char == 'H' || char == 'I' || char == 'U' || char == 'V' || char == 'W' || char == 'y' || char == 'Y' { - expectedRegex += "[0-9]{2}" - } else if char == 'M' { - expectedRegex += "[0-9]{2}" - } else if char == 'j' { - expectedRegex += "[0-9]{3}" - } else if char == 'S' || char == 'm' { - expectedRegex += "[0-9]{2}" - } else if char == 'c' { - // Note: Specific to the POSIX locale - expectedRegex += buildRegexFromTimeFormat("%a %b %e %H:%M:%S %Y") - } else if char == 'a' { - // Note: Specific to the POSIX locale - expectedRegex += "(Sun|Mon|Tue|Wed|Thu|Fri|Sat)" - } else if char == 'b' || char == 'h' { - // Note: Specific to the POSIX locale - expectedRegex += "(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)" - } else if char == 'e' || char == 'k' || char == 'l' { - expectedRegex += "[0-9 ]{2}" - } else if char == 'n' { - expectedRegex += "\n" - } else if char == 'p' { - expectedRegex += "(AM|PM)" - } else if char == 'P' { - expectedRegex += "(am|pm)" - } else if char == 's' { - expectedRegex += "\\d+" - } else if char == 'z' { - expectedRegex += "[+-][0-9]{4}" - } else if char == 'r' { - expectedRegex += buildRegexFromTimeFormat("%I:%M:%S %p") - } else if char == 'R' { - expectedRegex += buildRegexFromTimeFormat("%H:%M") - } else if char == 'x' { - expectedRegex += buildRegexFromTimeFormat("%m/%d/%y") - } else if char == 'X' { - expectedRegex += buildRegexFromTimeFormat("%H:%M:%S") - } else { - panic(fmt.Sprintf("buildRegexFromTimeFormat doesn't support %%%v, please open a bug against github.com/ddworken/hishtory", string(char))) - } - } else if char != '%' { - expectedRegex += regexp.QuoteMeta(string(char)) - } - lastCharWasPercent = false - if char == '%' { - lastCharWasPercent = true - } - } - return expectedRegex -} - -func maybeSkipBashHistTimePrefix(cmdLine string) (string, error) { - format := os.Getenv("HISTTIMEFORMAT") - if format == "" { - return cmdLine, nil - } - re, err := regexp.Compile("^" + buildRegexFromTimeFormat(format)) - if err != nil { - return "", fmt.Errorf("failed to parse regex for HISTTIMEFORMAT variable: %v", err) - } - return re.ReplaceAllLiteralString(cmdLine, ""), nil -} - -func ParseCrossPlatformInt(data string) (int64, error) { - data = strings.TrimSuffix(data, "N") - return strconv.ParseInt(data, 10, 64) -} - -func getLastCommand(history string) (string, error) { - split := strings.SplitN(strings.TrimSpace(history), " ", 2) - if len(split) <= 1 { - return "", fmt.Errorf("got unexpected bash history line: %#v, please open a bug at github.com/ddworken/hishtory", history) - } - split = strings.SplitN(split[1], " ", 2) - if len(split) <= 1 { - return "", fmt.Errorf("got unexpected bash history line: %#v, please open a bug at github.com/ddworken/hishtory", history) - } - return split[1], nil -} - -func shouldSkipHiddenCommand(ctx *context.Context, historyLine string) (bool, error) { - config := hctx.GetConf(ctx) - if config.LastSavedHistoryLine == historyLine { - return true, nil - } - config.LastSavedHistoryLine = historyLine - err := hctx.SetConfig(config) - if err != nil { - return false, err - } - return false, nil -} - func Setup(userSecret string, isOffline bool) error { if userSecret == "" { userSecret = uuid.Must(uuid.NewRandom()).String() @@ -532,6 +221,25 @@ func CheckFatalError(err error) { } } +func stripZshWeirdness(cmd string) string { + // Zsh has this weird behavior where sometimes commands are saved in the hishtory file + // with a weird prefix. I've never been able to figure out why this happens, but we + // can at least strip it. + firstCommandBugRegex := regexp.MustCompile(`: \d+:\d;(.*)`) + matches := firstCommandBugRegex.FindStringSubmatch(cmd) + if len(matches) == 2 { + return matches[1] + } + return cmd +} + +func isBashWeirdness(cmd string) bool { + // Bash has this weird behavior where the it has entries like `#1664342754` in the + // history file. We want to skip these. + firstCommandBugRegex := regexp.MustCompile(`^#\d+\s+$`) + return firstCommandBugRegex.MatchString(cmd) +} + func ImportHistory(ctx *context.Context, shouldReadStdin, force bool) (int, error) { config := hctx.GetConf(ctx) if config.HaveCompletedInitialImport && !force { diff --git a/client/lib/lib_test.go b/client/lib/lib_test.go index fbd4ff4..bf24441 100644 --- a/client/lib/lib_test.go +++ b/client/lib/lib_test.go @@ -2,10 +2,8 @@ package lib import ( "os" - "os/user" "path" "reflect" - "strings" "testing" "time" @@ -62,129 +60,6 @@ func TestSetupOffline(t *testing.T) { t.Fatalf("hishtory config should have been offline, actual=%#v", string(data)) } } - -func TestBuildHistoryEntry(t *testing.T) { - defer testutils.BackupAndRestore(t)() - defer testutils.RunTestServer()() - testutils.Check(t, Setup("", false)) - - // Test building an actual entry for bash - entry, err := BuildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "bash", "120", " 123 ls /foo ", "1641774958"}) - testutils.Check(t, err) - if entry.ExitCode != 120 { - t.Fatalf("history entry has unexpected exit code: %v", entry.ExitCode) - } - user, err := user.Current() - if err != nil { - t.Fatalf("failed to retrieve user: %v", err) - } - if entry.LocalUsername != user.Username { - t.Fatalf("history entry has unexpected user name: %v", entry.LocalUsername) - } - if !strings.HasPrefix(entry.CurrentWorkingDirectory, "/") && !strings.HasPrefix(entry.CurrentWorkingDirectory, "~/") { - t.Fatalf("history entry has unexpected cwd: %v", entry.CurrentWorkingDirectory) - } - if !strings.HasPrefix(entry.HomeDirectory, "/") { - t.Fatalf("history entry has unexpected home directory: %v", entry.HomeDirectory) - } - if entry.Command != "ls /foo" { - t.Fatalf("history entry has unexpected command: %v", entry.Command) - } - if !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-09T") && !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-10T") { - t.Fatalf("history entry has incorrect date in the start time: %v", entry.StartTime.Format(time.RFC3339)) - } - if entry.StartTime.Unix() != 1641774958 { - t.Fatalf("history entry has incorrect Unix time in the start time: %v", entry.StartTime.Unix()) - } - - // Test building an entry for zsh - entry, err = BuildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "zsh", "120", "ls /foo\n", "1641774958"}) - testutils.Check(t, err) - if entry.ExitCode != 120 { - t.Fatalf("history entry has unexpected exit code: %v", entry.ExitCode) - } - if entry.LocalUsername != user.Username { - t.Fatalf("history entry has unexpected user name: %v", entry.LocalUsername) - } - if !strings.HasPrefix(entry.CurrentWorkingDirectory, "/") && !strings.HasPrefix(entry.CurrentWorkingDirectory, "~/") { - t.Fatalf("history entry has unexpected cwd: %v", entry.CurrentWorkingDirectory) - } - if !strings.HasPrefix(entry.HomeDirectory, "/") { - t.Fatalf("history entry has unexpected home directory: %v", entry.HomeDirectory) - } - if entry.Command != "ls /foo" { - t.Fatalf("history entry has unexpected command: %v", entry.Command) - } - if !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-09T") && !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-10T") { - t.Fatalf("history entry has incorrect date in the start time: %v", entry.StartTime.Format(time.RFC3339)) - } - if entry.StartTime.Unix() != 1641774958 { - t.Fatalf("history entry has incorrect Unix time in the start time: %v", entry.StartTime.Unix()) - } - - // Test building an entry for fish - entry, err = BuildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "fish", "120", "ls /foo\n", "1641774958"}) - testutils.Check(t, err) - if entry.ExitCode != 120 { - t.Fatalf("history entry has unexpected exit code: %v", entry.ExitCode) - } - if entry.LocalUsername != user.Username { - t.Fatalf("history entry has unexpected user name: %v", entry.LocalUsername) - } - if !strings.HasPrefix(entry.CurrentWorkingDirectory, "/") && !strings.HasPrefix(entry.CurrentWorkingDirectory, "~/") { - t.Fatalf("history entry has unexpected cwd: %v", entry.CurrentWorkingDirectory) - } - if !strings.HasPrefix(entry.HomeDirectory, "/") { - t.Fatalf("history entry has unexpected home directory: %v", entry.HomeDirectory) - } - if entry.Command != "ls /foo" { - t.Fatalf("history entry has unexpected command: %v", entry.Command) - } - if !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-09T") && !strings.HasPrefix(entry.StartTime.Format(time.RFC3339), "2022-01-10T") { - t.Fatalf("history entry has incorrect date in the start time: %v", entry.StartTime.Format(time.RFC3339)) - } - if entry.StartTime.Unix() != 1641774958 { - t.Fatalf("history entry has incorrect Unix time in the start time: %v", entry.StartTime.Unix()) - } - - // Test building an entry that is empty, and thus not saved - entry, err = BuildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "zsh", "120", " \n", "1641774958"}) - testutils.Check(t, err) - if entry != nil { - t.Fatalf("expected history entry to be nil") - } -} - -func TestBuildHistoryEntryWithTimestampStripping(t *testing.T) { - defer testutils.BackupAndRestoreEnv("HISTTIMEFORMAT")() - defer testutils.BackupAndRestore(t)() - defer testutils.RunTestServer()() - testutils.Check(t, Setup("", false)) - - testcases := []struct { - input, histtimeformat, expectedCommand string - }{ - {" 123 ls /foo ", "", "ls /foo"}, - {" 2389 [2022-09-28 04:38:32 +0000] echo", "", "[2022-09-28 04:38:32 +0000] echo"}, - {" 2389 [2022-09-28 04:38:32 +0000] echo", "[%F %T %z] ", "echo"}, - } - for _, tc := range testcases { - conf := hctx.GetConf(hctx.MakeContext()) - conf.LastSavedHistoryLine = "" - testutils.Check(t, hctx.SetConfig(conf)) - - os.Setenv("HISTTIMEFORMAT", tc.histtimeformat) - entry, err := BuildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "bash", "120", tc.input, "1641774958"}) - testutils.Check(t, err) - if entry == nil { - t.Fatalf("entry is unexpectedly nil") - } - if entry.Command != tc.expectedCommand { - t.Fatalf("BuildHistoryEntry(%#v) returned %#v (expected=%#v)", tc.input, entry.Command, tc.expectedCommand) - } - } -} - func TestPersist(t *testing.T) { defer testutils.BackupAndRestore(t)() testutils.Check(t, hctx.InitConfig()) @@ -317,99 +192,6 @@ func TestAddToDbIfNew(t *testing.T) { } } -func TestParseCrossPlatformInt(t *testing.T) { - res, err := ParseCrossPlatformInt("123") - testutils.Check(t, err) - if res != 123 { - t.Fatalf("failed to parse cross platform int %d", res) - } - res, err = ParseCrossPlatformInt("123N") - testutils.Check(t, err) - if res != 123 { - t.Fatalf("failed to parse cross platform int %d", res) - } -} - -func TestBuildRegexFromTimeFormat(t *testing.T) { - testcases := []struct { - formatString, regex string - }{ - {"%Y ", "[0-9]{4} "}, - {"%F %T ", "[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} "}, - {"%F%T", "[0-9]{4}-[0-9]{2}-[0-9]{2}[0-9]{2}:[0-9]{2}:[0-9]{2}"}, - {"%%", "%"}, - {"%%%%", "%%"}, - {"%%%Y", "%[0-9]{4}"}, - {"%%%F%T", "%[0-9]{4}-[0-9]{2}-[0-9]{2}[0-9]{2}:[0-9]{2}:[0-9]{2}"}, - } - - for _, tc := range testcases { - if regex := buildRegexFromTimeFormat(tc.formatString); regex != tc.regex { - t.Fatalf("building a regex for %#v returned %#v (expected=%#v)", tc.formatString, regex, tc.regex) - } - } -} - -func TestGetLastCommand(t *testing.T) { - testcases := []struct { - input, expectedOutput string - }{ - {" 0 ls", "ls"}, - {" 33 ls", "ls"}, - {" 33 ls --aaaa foo bar ", "ls --aaaa foo bar"}, - {" 2389 [2022-09-28 04:38:32 +0000] echo", "[2022-09-28 04:38:32 +0000] echo"}, - } - for _, tc := range testcases { - actualOutput, err := getLastCommand(tc.input) - testutils.Check(t, err) - if actualOutput != tc.expectedOutput { - t.Fatalf("getLastCommand(%#v) returned %#v (expected=%#v)", tc.input, actualOutput, tc.expectedOutput) - } - } -} - -func TestMaybeSkipBashHistTimePrefix(t *testing.T) { - defer testutils.BackupAndRestoreEnv("HISTTIMEFORMAT")() - - testcases := []struct { - env, cmdLine, expected string - }{ - {"%F %T ", "2019-07-12 13:02:31 sudo apt update", "sudo apt update"}, - {"%F %T ", "2019-07-12 13:02:31 ls a b", "ls a b"}, - {"%F %T ", "2019-07-12 13:02:31 ls a ", "ls a "}, - {"%F %T ", "2019-07-12 13:02:31 ls a", "ls a"}, - {"%F %T ", "2019-07-12 13:02:31 ls", "ls"}, - {"%F %T ", "2019-07-12 13:02:31 ls -Slah", "ls -Slah"}, - {"%F ", "2019-07-12 ls -Slah", "ls -Slah"}, - {"%F ", "2019-07-12 ls -Slah", "ls -Slah"}, - {"%Y", "2019ls -Slah", "ls -Slah"}, - {"%Y%Y", "20192020ls -Slah", "ls -Slah"}, - {"%Y%Y", "20192020ls -Slah20192020", "ls -Slah20192020"}, - {"", "ls -Slah", "ls -Slah"}, - {"[%F %T] ", "[2019-07-12 13:02:31] sudo apt update", "sudo apt update"}, - {"[%F a %T] ", "[2019-07-12 a 13:02:31] sudo apt update", "sudo apt update"}, - {"aaa ", "aaa sudo apt update", "sudo apt update"}, - {"%c ", "Sun Aug 19 02:56:02 2012 sudo apt update", "sudo apt update"}, - {"%c ", "Sun Aug 19 02:56:02 2012 ls", "ls"}, - {"[%c] ", "[Sun Aug 19 02:56:02 2012] ls", "ls"}, - {"[%c %t] ", "[Sun Aug 19 02:56:02 2012 ] ls", "ls"}, - {"[%c %t]", "[Sun Aug 19 02:56:02 2012 ]ls", "ls"}, - {"[%c %t]", "[Sun Aug 19 02:56:02 2012 ]foo", "foo"}, - {"[%c %t", "[Sun Aug 19 02:56:02 2012 foo", "foo"}, - {"[%F %T %z]", "[2022-09-28 04:17:06 +0000]foo", "foo"}, - {"[%F %T %z] ", "[2022-09-28 04:17:06 +0000] foo", "foo"}, - } - - for _, tc := range testcases { - os.Setenv("HISTTIMEFORMAT", tc.env) - stripped, err := maybeSkipBashHistTimePrefix(tc.cmdLine) - testutils.Check(t, err) - if stripped != tc.expected { - t.Fatalf("skipping the time prefix returned %#v (expected=%#v for %#v)", stripped, tc.expected, tc.cmdLine) - } - } -} - func TestChunks(t *testing.T) { testcases := []struct { input []int