From 3a8243d25d957ea97962f9d43f9afd23ce575894 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Fri, 8 Sep 2023 10:09:01 -0700 Subject: [PATCH 01/13] Another set of extra delays for testTui/search, similar to 37c06a31db8b3600fd83525a45b2a741e29e1ade --- client/client_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 507ce36..7f2c8d2 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -1932,9 +1932,9 @@ func testTui_search(t testing.TB) { } // Check the output when the initial search is invalid - out = captureTerminalOutput(t, tester, []string{ - "hishtory SPACE tquery SPACE foo: ENTER", - "ls", + out = captureTerminalOutputWithComplexCommands(t, tester, []TmuxCommand{ + {Keys: "hishtory SPACE tquery SPACE foo: ENTER", ExtraDelay: 1.0}, + {Keys: "ls", ExtraDelay: 1.0}, }) out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) testutils.CompareGoldens(t, out, "TestTui-InitialInvalidSearch") From 084af932ea060abca7fd6405d3dde5afdac08f08 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Fri, 8 Sep 2023 18:05:11 -0700 Subject: [PATCH 02/13] Update docker-compose test to hopefully run and properly record a command in hishtory --- .github/workflows/docker-compose-test.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-compose-test.yml b/.github/workflows/docker-compose-test.yml index 537a5aa..7e07e7a 100644 --- a/.github/workflows/docker-compose-test.yml +++ b/.github/workflows/docker-compose-test.yml @@ -36,14 +36,12 @@ jobs: set -eo pipefail export HISHTORY_SERVER=http://localhost source ~/.bashrc - # Record a command that we'll check was persisted - ls -Slah / - # Assert that the entry is syncing properly - ./hishtory status -v | grep 'Sync Status: Synced' # Check that hishtory query runs without errors ./hishtory query - # Check that hishtory export recorded the above ls command - ./hishtory export | grep "ls -Slah /" + # And check that entries get recorded properly + echo -e 'ls -Slah /\nhishtory export\n' | zsh -is | grep "ls -Slah /" + # Assert that the entry is syncing properly + ./hishtory status -v | grep 'Sync Status: Synced' - name: Setup tmate session if: ${{ failure() }} uses: mxschmitt/action-tmate@v3 From 27d5c592634b747f2efba45eb87699d92b71beda Mon Sep 17 00:00:00 2001 From: David Dworken Date: Fri, 8 Sep 2023 19:27:16 -0700 Subject: [PATCH 03/13] Skip datadog integratio on macos if colima fails to start, since colima appears to be flaky due to a code signing error --- .github/workflows/go-test.yml | 4 +++- client/client_test.go | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index eaa1bd8..ae4e938 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -43,7 +43,9 @@ jobs: # Set a consistent hostname so we can run tests that depend on it sudo scutil --set HostName ghaction-runner-hostname - + - name: MacOS Docker Setup + if: ${{ !startsWith(github.event.head_commit.message, 'Release') && matrix.os == 'macos-latest'}} + continue-on-error: true # Install docker so it can be used for datadog brew install docker colima start diff --git a/client/client_test.go b/client/client_test.go index 7f2c8d2..90fd17a 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -58,7 +58,12 @@ func TestMain(m *testing.M) { if _, has_dd_api_key := os.LookupEnv("DD_API_KEY"); testutils.IsGithubAction() && has_dd_api_key { ddStats, err := statsd.New("localhost:8125") if err != nil { - panic(fmt.Errorf("Failed to start DataDog statsd: %w\n", err)) + err := fmt.Errorf("Failed to start DataDog statsd: %w\n", err) + if runtime.GOOS == "darwin" { + fmt.Printf("%v", err) + } else { + panic(err) + } } GLOBAL_STATSD = ddStats } From ba099c384b61cf220d539b33654160be9c0f9a41 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Fri, 8 Sep 2023 19:40:10 -0700 Subject: [PATCH 04/13] Add statsd incr for overall test failure --- client/client_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/client_test.go b/client/client_test.go index 90fd17a..a0253f9 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -206,7 +206,7 @@ func runTestsWithRetries(parentT *testing.T, testName string, testFunc func(t te func runTestsWithExtraRetries(parentT *testing.T, testName string, testFunc func(t testing.TB), numRetries int) { for i := 1; i <= numRetries; i++ { - rt := &retryingTester{nil, i == numRetries, true} + rt := &retryingTester{nil, i == numRetries, true, testName} parentT.Run(fmt.Sprintf("%s/%d", testName, i), func(t *testing.T) { rt.T = t testFunc(rt) @@ -229,12 +229,16 @@ type retryingTester struct { *testing.T isFinalRun bool succeeded bool + testName string } func (t *retryingTester) Fatalf(format string, args ...any) { t.T.Helper() t.succeeded = false if t.isFinalRun { + if GLOBAL_STATSD != nil { + GLOBAL_STATSD.Incr("test_failure", []string{"test:" + t.testName, "os:" + runtime.GOOS}, 1.0) + } t.T.Fatalf(format, args...) } else { testutils.TestLog(t.T, fmt.Sprintf("retryingTester: Ignoring fatalf for non-final run: %#v", fmt.Sprintf(format, args...))) From 392ebfb0fda258787cc7addcf6c50e90619ca9cc Mon Sep 17 00:00:00 2001 From: David Dworken Date: Fri, 8 Sep 2023 19:41:42 -0700 Subject: [PATCH 05/13] Fix yaml syntax error introduced in 27d5c592634b747f2efba45eb87699d92b71beda --- .github/workflows/go-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index ae4e938..eceab0d 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -46,6 +46,7 @@ jobs: - name: MacOS Docker Setup if: ${{ !startsWith(github.event.head_commit.message, 'Release') && matrix.os == 'macos-latest'}} continue-on-error: true + run: | # Install docker so it can be used for datadog brew install docker colima start From 4d7e685a6e5485bef5cc5993dba592b616a64a73 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Fri, 8 Sep 2023 19:44:36 -0700 Subject: [PATCH 06/13] Add ExtraDelay to make testTui/delete more reliable --- client/client_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index a0253f9..daebc0c 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -1875,10 +1875,11 @@ func testTui_delete(t testing.TB) { manuallySubmitHistoryEntry(t, userSecret, testutils.MakeFakeHistoryEntry("echo 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'")) // Check that we can delete an entry - out := captureTerminalOutput(t, tester, []string{ - "hishtory SPACE tquery ENTER", - "aaaaaa", - "C-K", + out := captureTerminalOutputWithComplexCommands(t, tester, []TmuxCommand{ + {Keys: "hishtory SPACE tquery ENTER"}, + // ExtraDelay so that the search query finishes before we hit delete + {Keys: "aaaaaa", ExtraDelay: 1.0}, + {Keys: "C-K"}, }) out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) testutils.CompareGoldens(t, out, "TestTui-Delete") From b6b4499a429783eacf5ac02bb57cbff8f218d75d Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sat, 9 Sep 2023 11:50:07 -0700 Subject: [PATCH 07/13] Add a few more ExtraDelays + comments to further improve test reliability --- client/client_test.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index daebc0c..b366884 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -1822,7 +1822,7 @@ func testTui_resize(t testing.TB) { out = captureTerminalOutputWithShellNameAndDimensions(t, tester, tester.ShellName(), 100, 20, []TmuxCommand{ {Keys: "hishtory SPACE tquery ENTER"}, {Keys: "Down"}, - {ResizeX: 300, ResizeY: 100}, + {ResizeX: 300, ResizeY: 100, ExtraDelay: 1.0}, {Keys: "Enter"}, }) require.Contains(t, out, "\necho 'aaaaaa bbbb'\n") @@ -1931,9 +1931,11 @@ func testTui_search(t testing.TB) { testutils.CompareGoldens(t, out, "TestTui-Search") // Check the output when there is a selected result - out = captureTerminalOutput(t, tester, []string{ - "hishtory SPACE tquery ENTER", - "ls", "", "ENTER", + out = captureTerminalOutputWithComplexCommands(t, tester, []TmuxCommand{ + {Keys: "hishtory SPACE tquery ENTER"}, + // Extra delay to ensure that the search for 'ls' finishes before we select an entry + {Keys: "ls", ExtraDelay: 1.0}, + {Keys: "ENTER"}, }) out = strings.Split(strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]), "\n")[0] expected = `ls ~/` @@ -1943,6 +1945,7 @@ func testTui_search(t testing.TB) { // Check the output when the initial search is invalid out = captureTerminalOutputWithComplexCommands(t, tester, []TmuxCommand{ + // ExtraDelay to ensure that after searching for 'foo:' it gets the real results for an empty query {Keys: "hishtory SPACE tquery SPACE foo: ENTER", ExtraDelay: 1.0}, {Keys: "ls", ExtraDelay: 1.0}, }) @@ -1952,7 +1955,9 @@ func testTui_search(t testing.TB) { // Check the output when the search is invalid out = captureTerminalOutputWithComplexCommands(t, tester, []TmuxCommand{ {Keys: "hishtory SPACE tquery ENTER"}, - {Keys: "ls", ExtraDelay: 1.0}, {Keys: ":"}, + // ExtraDelay to ensure that the search for 'ls' finishes before we make it invalid by adding ':' + {Keys: "ls", ExtraDelay: 1.0}, + {Keys: ":"}, }) out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) testutils.CompareGoldens(t, out, "TestTui-InvalidSearch") From 223598c8f2adfd868ca4c519dc423eeaee50538f Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sat, 9 Sep 2023 11:56:06 -0700 Subject: [PATCH 08/13] Remove golden that was forked for actions, that didn't need to be --- client/client_test.go | 6 +--- .../TestTimestampFormat-tquery-isAction | 31 ------------------- 2 files changed, 1 insertion(+), 36 deletions(-) delete mode 100644 client/lib/goldens/TestTimestampFormat-tquery-isAction diff --git a/client/client_test.go b/client/client_test.go index b366884..d495f4e 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -2400,11 +2400,7 @@ func TestTimestampFormat(t *testing.T) { testutils.CompareGoldens(t, out, "TestTimestampFormat-query") out = captureTerminalOutput(t, tester, []string{"hishtory SPACE tquery SPACE -pipefail SPACE -tablesizing ENTER"}) out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) - goldenName := "TestTimestampFormat-tquery" - if testutils.IsGithubAction() { - goldenName += "-isAction" - } - testutils.CompareGoldens(t, out, goldenName) + testutils.CompareGoldens(t, out, "TestTimestampFormat-tquery") } func TestZDotDir(t *testing.T) { diff --git a/client/lib/goldens/TestTimestampFormat-tquery-isAction b/client/lib/goldens/TestTimestampFormat-tquery-isAction deleted file mode 100644 index 31fc97d..0000000 --- a/client/lib/goldens/TestTimestampFormat-tquery-isAction +++ /dev/null @@ -1,31 +0,0 @@ --pipefail -tablesizing - - - -Search Query: > -pipefail -tablesizing - -┌───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ Hostname CWD Timestamp Runtime Exit Code Command │ -│───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────│ -│ localhost ~/foo/ 2022/Apr/16 01:03 24s 3 table_cmd2 │ -│ localhost /tmp/ 2022/Apr/16 01:03 4s 2 table_cmd1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa… │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -└───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ -hiSHtory: Search your shell history • ctrl+h help \ No newline at end of file From 2ccbb8d58721e30afd2c6e2ad92326377a2656a3 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sat, 9 Sep 2023 12:28:01 -0700 Subject: [PATCH 09/13] Store timestamps in DB in Unix time, and display them in the Local time for consistent TZs in displays, and to ensure ordering is correct This fixes a bug where entries could be out of order if they were recorded on machines in different TZs (e.g. if one machine had the TZ set to UTC). Admittedly, this commit isn't a perfect fix since it doesn't change all the existing hishtory entries to be stored in UTC, but this just means that the ordering bug will continue to exist for historical entries, but be fixed for new entries. And for our purposes, that is good enough. :) # Please enter the commit message for your changes. Lines starting --- client/client_test.go | 86 ++++++++++--------- client/cmd/saveHistoryEntry.go | 8 +- .../TestSortByConsistentTimezone-query | 4 + .../TestSortByConsistentTimezone-tquery | 31 +++++++ client/lib/lib.go | 20 +++-- shared/testutils/testutils.go | 4 +- 6 files changed, 101 insertions(+), 52 deletions(-) create mode 100644 client/lib/goldens/TestSortByConsistentTimezone-query create mode 100644 client/lib/goldens/TestSortByConsistentTimezone-tquery diff --git a/client/client_test.go b/client/client_test.go index d495f4e..ea30926 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -429,13 +429,9 @@ func testBasicUserFlow(t *testing.T, tester shellTester, onlineStatus OnlineStat // Assert that hishtory is correctly using the dev config.sh homedir, err := os.UserHomeDir() - if err != nil { - t.Fatalf("failed to get homedir: %v", err) - } + require.NoError(t, err) dat, err := os.ReadFile(path.Join(homedir, data.GetHishtoryPath(), "config.sh")) - if err != nil { - t.Fatalf("failed to read config.sh: %v", err) - } + require.NoError(t, err, "failed to read config.sh") require.NotContains(t, string(dat), "# Background Run", "config.sh is the prod version when it shouldn't be") // Test the banner @@ -740,9 +736,7 @@ func testUpdate(t *testing.T, tester shellTester) { // Update out = tester.RunInteractiveShell(t, `hishtory update`) isExpected, err := regexp.MatchString(`Successfully updated hishtory from v0[.]Unknown to v0.\d+`, out) - if err != nil { - t.Fatalf("regex failure: %v", err) - } + require.NoError(t, err, "regex failure") if !isExpected { t.Fatalf("hishtory update returned unexpected out=%#v", out) } @@ -1019,13 +1013,9 @@ CGO_ENABLED=0 go build -o /tmp/client // Assert that config.sh isn't the dev version homedir, err := os.UserHomeDir() - if err != nil { - t.Fatalf("failed to get homedir: %v", err) - } + require.NoError(t, err) dat, err := os.ReadFile(path.Join(homedir, data.GetHishtoryPath(), "config.sh")) - if err != nil { - t.Fatalf("failed to read config.sh: %v", err) - } + require.NoError(t, err, "failed to read config.sh") require.NotContains(t, string(dat), "except it doesn't run the save process in the background", "config.sh is the testing version when it shouldn't be") // Test the status subcommand @@ -1065,9 +1055,7 @@ func testDisplayTable(t *testing.T, tester shellTester) { // Submit two fake entries tmz, err := time.LoadLocation("America/Los_Angeles") - if err != nil { - t.Fatalf("failed to load timezone: %v", err) - } + require.NoError(t, err) entry1 := testutils.MakeFakeHistoryEntry("table_cmd1") entry1.StartTime = time.Unix(1650096186, 0).In(tmz) entry1.EndTime = time.Unix(1650096190, 0).In(tmz) @@ -1131,9 +1119,7 @@ func testRequestAndReceiveDbDump(t *testing.T, tester shellTester) { config := hctx.GetConf(hctx.MakeContext()) deviceId1 := config.DeviceId resp, err := lib.ApiGet("/api/v1/get-dump-requests?user_id=" + data.UserId(secretKey) + "&device_id=" + deviceId1) - if err != nil { - t.Fatalf("failed to get pending dump requests: %v", err) - } + require.NoError(t, err, "failed to get pending dump requests") if string(resp) != "[]" { t.Fatalf("There are pending dump requests! user_id=%#v, resp=%#v", data.UserId(secretKey), string(resp)) } @@ -1158,18 +1144,14 @@ echo other`) // Wipe the DB to simulate entries getting deleted because they've already been read and expired _, err = lib.ApiGet("/api/v1/wipe-db-entries") - if err != nil { - t.Fatalf("failed to wipe the DB: %v", err) - } + require.NoError(t, err, "failed to wipe the remote DB") // Install a new one (with the same secret key but a diff device id) installHishtory(t, tester, secretKey) // Confirm there is now a pending dump requests that the first device should respond to resp, err = lib.ApiGet("/api/v1/get-dump-requests?user_id=" + data.UserId(secretKey) + "&device_id=" + deviceId1) - if err != nil { - t.Fatalf("failed to get pending dump requests: %v", err) - } + require.NoError(t, err, "failed to get pending dump requests") if string(resp) == "[]" { t.Fatalf("There are no pending dump requests! user_id=%#v, resp=%#v", data.UserId(secretKey), string(resp)) } @@ -1200,9 +1182,7 @@ echo other`) // Confirm there are no pending dump requests for the first device resp, err = lib.ApiGet("/api/v1/get-dump-requests?user_id=" + data.UserId(secretKey) + "&device_id=" + deviceId1) - if err != nil { - t.Fatalf("failed to get pending dump requests: %v", err) - } + require.NoError(t, err, "failed to get pending dump requests") if string(resp) != "[]" { t.Fatalf("There are pending dump requests! user_id=%#v, resp=%#v", data.UserId(secretKey), string(resp)) } @@ -2378,9 +2358,7 @@ func TestTimestampFormat(t *testing.T) { // Add some entries with fixed timestamps tmz, err := time.LoadLocation("America/Los_Angeles") - if err != nil { - t.Fatalf("failed to load timezone: %v", err) - } + require.NoError(t, err) entry1 := testutils.MakeFakeHistoryEntry("table_cmd1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") entry1.StartTime = time.Unix(1650096186, 0).In(tmz) entry1.EndTime = time.Unix(1650096190, 0).In(tmz) @@ -2403,6 +2381,40 @@ func TestTimestampFormat(t *testing.T) { testutils.CompareGoldens(t, out, "TestTimestampFormat-tquery") } +func TestSortByConsistentTimezone(t *testing.T) { + // Setup + tester := zshTester{} + defer testutils.BackupAndRestore(t)() + installHishtory(t, tester, "") + + // Add some entries with timestamps in different timezones + db := hctx.GetDb(hctx.MakeContext()) + timestamp := int64(1650096186) + la_time, err := time.LoadLocation("America/Los_Angeles") + require.NoError(t, err) + ny_time, err := time.LoadLocation("America/New_York") + require.NoError(t, err) + entry1 := testutils.MakeFakeHistoryEntry("first_entry") + entry1.StartTime = time.Unix(timestamp, 0).In(ny_time) + entry1.EndTime = time.Unix(timestamp+1, 0).In(ny_time) + testutils.Check(t, lib.ReliableDbCreate(db, entry1)) + entry2 := testutils.MakeFakeHistoryEntry("second_entry") + entry2.StartTime = time.Unix(timestamp+1000, 0).In(la_time) + entry2.EndTime = time.Unix(timestamp+1001, 0).In(la_time) + testutils.Check(t, lib.ReliableDbCreate(db, entry2)) + entry3 := testutils.MakeFakeHistoryEntry("third_entry") + entry3.StartTime = time.Unix(timestamp+2000, 0).In(ny_time) + entry3.EndTime = time.Unix(timestamp+2001, 0).In(ny_time) + testutils.Check(t, lib.ReliableDbCreate(db, entry3)) + + // And check that they're displayed in the correct order + out := hishtoryQuery(t, tester, "-pipefail") + testutils.CompareGoldens(t, out, "TestSortByConsistentTimezone-query") + out = captureTerminalOutput(t, tester, []string{"hishtory SPACE tquery SPACE -pipefail ENTER"}) + out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) + testutils.CompareGoldens(t, out, "TestSortByConsistentTimezone-tquery") +} + func TestZDotDir(t *testing.T) { // Setup tester := zshTester{} @@ -2493,14 +2505,10 @@ func TestSetConfigNoCorruption(t *testing.T) { c.HaveMissedUploads = (i % 2) == 0 // Write it err := hctx.SetConfig(c) - if err != nil { - panic(err) - } + require.NoError(t, err) // Check that we can read c2, err := hctx.GetConfig() - if err != nil { - panic(err) - } + require.NoError(t, err) if c2.UserSecret != c.UserSecret { panic("user secret mismatch") } diff --git a/client/cmd/saveHistoryEntry.go b/client/cmd/saveHistoryEntry.go index e5e913c..4840281 100644 --- a/client/cmd/saveHistoryEntry.go +++ b/client/cmd/saveHistoryEntry.go @@ -105,8 +105,8 @@ func presaveHistoryEntry(ctx context.Context) { } startTime, err := parseCrossPlatformInt(os.Args[4]) lib.CheckFatalError(err) - entry.StartTime = time.Unix(startTime, 0) - entry.EndTime = time.Unix(0, 0) + entry.StartTime = time.Unix(startTime, 0).UTC() + entry.EndTime = time.Unix(0, 0).UTC() // And persist it locally. db := hctx.GetDb(ctx) @@ -281,10 +281,10 @@ func buildHistoryEntry(ctx context.Context, args []string) (*data.HistoryEntry, if err != nil { return nil, fmt.Errorf("failed to parse start time %s as int: %w", args[5], err) } - entry.StartTime = time.Unix(seconds, 0) + entry.StartTime = time.Unix(seconds, 0).UTC() // end time - entry.EndTime = time.Now() + entry.EndTime = time.Now().UTC() // command if shell == "bash" { diff --git a/client/lib/goldens/TestSortByConsistentTimezone-query b/client/lib/goldens/TestSortByConsistentTimezone-query new file mode 100644 index 0000000..1f793d6 --- /dev/null +++ b/client/lib/goldens/TestSortByConsistentTimezone-query @@ -0,0 +1,4 @@ +Hostname CWD Timestamp Runtime Exit Code Command +localhost /tmp/ Apr 16 2022 01:36:26 PDT 1s 2 third_entry +localhost /tmp/ Apr 16 2022 01:19:46 PDT 1s 2 second_entry +localhost /tmp/ Apr 16 2022 01:03:06 PDT 1s 2 first_entry diff --git a/client/lib/goldens/TestSortByConsistentTimezone-tquery b/client/lib/goldens/TestSortByConsistentTimezone-tquery new file mode 100644 index 0000000..cf7438c --- /dev/null +++ b/client/lib/goldens/TestSortByConsistentTimezone-tquery @@ -0,0 +1,31 @@ +-pipefail + + + +Search Query: > -pipefail + +┌──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ Hostname CWD Timestamp Runtime Exit Code Command │ +│──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────│ +│ localhost /tmp/ Apr 16 2022 01:36:26 PDT 1s 2 third_entry │ +│ localhost /tmp/ Apr 16 2022 01:19:46 PDT 1s 2 second_entry │ +│ localhost /tmp/ Apr 16 2022 01:03:06 PDT 1s 2 first_entry │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +hiSHtory: Search your shell history • ctrl+h help \ No newline at end of file diff --git a/client/lib/lib.go b/client/lib/lib.go index eb7932d..cf9e611 100644 --- a/client/lib/lib.go +++ b/client/lib/lib.go @@ -117,7 +117,7 @@ func AddToDbIfNew(db *gorm.DB, entry data.HistoryEntry) { var results []data.HistoryEntry tx.Limit(1).Find(&results) if len(results) == 0 { - db.Create(entry) + db.Create(normalizeEntryTimezone(entry)) // TODO: check the error here and bubble it up } } @@ -146,13 +146,13 @@ func buildTableRow(ctx context.Context, columnNames []string, entry data.History case "CWD": row = append(row, entry.CurrentWorkingDirectory) case "Timestamp": - row = append(row, entry.StartTime.Format(hctx.GetConf(ctx).TimestampFormat)) + row = append(row, entry.StartTime.Local().Format(hctx.GetConf(ctx).TimestampFormat)) case "Runtime": if entry.EndTime == time.Unix(0, 0) { // An EndTime of zero means this is a pre-saved entry that never finished row = append(row, "N/A") } else { - row = append(row, entry.EndTime.Sub(entry.StartTime).Round(time.Millisecond).String()) + row = append(row, entry.EndTime.Local().Sub(entry.StartTime.Local()).Round(time.Millisecond).String()) } case "Exit Code": row = append(row, fmt.Sprintf("%d", entry.ExitCode)) @@ -301,8 +301,8 @@ func ImportHistory(ctx context.Context, shouldReadStdin, force bool) (int, error CurrentWorkingDirectory: "Unknown", HomeDirectory: homedir, ExitCode: 0, - StartTime: time.Now(), - EndTime: time.Now(), + StartTime: time.Now().UTC(), + EndTime: time.Now().UTC(), DeviceId: config.DeviceId, } err = ReliableDbCreate(db, entry) @@ -699,7 +699,14 @@ func IsOfflineError(err error) bool { strings.Contains(err.Error(), "net/http: TLS handshake timeout") } -func ReliableDbCreate(db *gorm.DB, entry interface{}) error { +func normalizeEntryTimezone(entry data.HistoryEntry) data.HistoryEntry { + entry.StartTime = entry.StartTime.UTC() + entry.EndTime = entry.EndTime.UTC() + return entry +} + +func ReliableDbCreate(db *gorm.DB, entry data.HistoryEntry) error { + entry = normalizeEntryTimezone(entry) var err error = nil i := 0 for i = 0; i < 10; i++ { @@ -892,7 +899,6 @@ func Search(ctx context.Context, db *gorm.DB, query string, limit int) ([]*data. if err != nil { return nil, err } - // TODO: This ordering isn't sufficient if some computers are in different timezones. Add better sorting here. if hctx.GetConf(ctx).BetaMode { tx = tx.Order("start_time DESC") } else { diff --git a/shared/testutils/testutils.go b/shared/testutils/testutils.go index 4ad1247..77ec18a 100644 --- a/shared/testutils/testutils.go +++ b/shared/testutils/testutils.go @@ -311,8 +311,8 @@ func MakeFakeHistoryEntry(command string) data.HistoryEntry { CurrentWorkingDirectory: "/tmp/", HomeDirectory: "/home/david/", ExitCode: 2, - StartTime: time.Unix(fakeHistoryTimestamp, 0), - EndTime: time.Unix(fakeHistoryTimestamp+3, 0), + StartTime: time.Unix(fakeHistoryTimestamp, 0).UTC(), + EndTime: time.Unix(fakeHistoryTimestamp+3, 0).UTC(), } } From 29723723bcc7f2bf8eac87093677a89dfa26bafe Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sat, 9 Sep 2023 12:36:02 -0700 Subject: [PATCH 10/13] Fix link to prebuilt server binaries --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 979985c..4ea79f5 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ Download the latest binary from [Github Releases](https://github.com/ddworken/hi By default, hiSHtory relies on a backend for syncing. All data is end-to-end encrypted, so the backend can't view your history. -But if you'd like to self-host the hishtory backend, you can! The backend is a simple go binary in `backend/server/server.go` (with [prebuilt binaries here](https://github.com/ddworken/hishtory/releases)). It can either use SQLite or Postgres for persistence. +But if you'd like to self-host the hishtory backend, you can! The backend is a simple go binary in `backend/server/server.go` (with [prebuilt binaries here](https://github.com/ddworken/hishtory/tags)). It can either use SQLite or Postgres for persistence. Check out the [`docker-compose.yml`](https://github.com/ddworken/hishtory/blob/master/backend/server/docker-compose.yml) file for an example config to start a hiSHtory server using postgres. From e3e3a7bd0dda91d3d45ac013cb9c98c47ce89ee1 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sat, 9 Sep 2023 14:02:47 -0700 Subject: [PATCH 11/13] Add tablesizing command to get consistent table dimensions --- client/client_test.go | 7 ++- .../TestSortByConsistentTimezone-tquery | 52 +++++++++---------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index ea30926..ae0d6f8 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -2387,6 +2387,9 @@ func TestSortByConsistentTimezone(t *testing.T) { defer testutils.BackupAndRestore(t)() installHishtory(t, tester, "") + // Add an entry just to ensure we get consistent table sizing + tester.RunInteractiveShell(t, "echo tablesizing") + // Add some entries with timestamps in different timezones db := hctx.GetDb(hctx.MakeContext()) timestamp := int64(1650096186) @@ -2408,9 +2411,9 @@ func TestSortByConsistentTimezone(t *testing.T) { testutils.Check(t, lib.ReliableDbCreate(db, entry3)) // And check that they're displayed in the correct order - out := hishtoryQuery(t, tester, "-pipefail") + out := hishtoryQuery(t, tester, "-pipefail -tablesizing") testutils.CompareGoldens(t, out, "TestSortByConsistentTimezone-query") - out = captureTerminalOutput(t, tester, []string{"hishtory SPACE tquery SPACE -pipefail ENTER"}) + out = captureTerminalOutput(t, tester, []string{"hishtory SPACE tquery SPACE -pipefail SPACE -tablesizing ENTER"}) out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) testutils.CompareGoldens(t, out, "TestSortByConsistentTimezone-tquery") } diff --git a/client/lib/goldens/TestSortByConsistentTimezone-tquery b/client/lib/goldens/TestSortByConsistentTimezone-tquery index cf7438c..83f2331 100644 --- a/client/lib/goldens/TestSortByConsistentTimezone-tquery +++ b/client/lib/goldens/TestSortByConsistentTimezone-tquery @@ -1,31 +1,31 @@ --pipefail +-pipefail -tablesizing -Search Query: > -pipefail +Search Query: > -pipefail -tablesizing -┌──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ Hostname CWD Timestamp Runtime Exit Code Command │ -│──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────│ -│ localhost /tmp/ Apr 16 2022 01:36:26 PDT 1s 2 third_entry │ -│ localhost /tmp/ Apr 16 2022 01:19:46 PDT 1s 2 second_entry │ -│ localhost /tmp/ Apr 16 2022 01:03:06 PDT 1s 2 first_entry │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -│ │ -└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +┌────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ Hostname CWD Timestamp Runtime Exit Code Command │ +│────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────│ +│ localhost /tmp/ Apr 16 2022 01:36:26 PDT 1s 2 third_entry │ +│ localhost /tmp/ Apr 16 2022 01:19:46 PDT 1s 2 second_entry │ +│ localhost /tmp/ Apr 16 2022 01:03:06 PDT 1s 2 first_entry │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ hiSHtory: Search your shell history • ctrl+h help \ No newline at end of file From c269e7832e06795848d340cdfc00369892a6e223 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sat, 9 Sep 2023 15:17:55 -0700 Subject: [PATCH 12/13] Release v0.212 --- VERSION | 2 +- client/client_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index dba40af..0d38910 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -211 +212 diff --git a/client/client_test.go b/client/client_test.go index ae0d6f8..28035bd 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -186,7 +186,7 @@ func TestParam(t *testing.T) { } runTestsWithRetries(t, "testControlR/offline/bash", func(t testing.TB) { testControlR(t, bashTester{}, "bash", Offline) }) runTestsWithRetries(t, "testControlR/fish", func(t testing.TB) { testControlR(t, bashTester{}, "fish", Online) }) - runTestsWithExtraRetries(t, "testTui/search", testTui_search, 7) + runTestsWithExtraRetries(t, "testTui/search", testTui_search, 10) runTestsWithRetries(t, "testTui/general", testTui_general) runTestsWithRetries(t, "testTui/scroll", testTui_scroll) runTestsWithRetries(t, "testTui/resize", testTui_resize) From 9b871e93ed5f0bcee781d3ebb3c7c756aa838737 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Sun, 10 Sep 2023 17:13:48 -0700 Subject: [PATCH 13/13] Fix consistent time zone sorting test to account for table sizing differences on github actions --- client/client_test.go | 2 +- ...yConsistentTimezone-tquery-isAction=false} | 0 ...tByConsistentTimezone-tquery-isAction=true | 31 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) rename client/lib/goldens/{TestSortByConsistentTimezone-tquery => TestSortByConsistentTimezone-tquery-isAction=false} (100%) create mode 100644 client/lib/goldens/TestSortByConsistentTimezone-tquery-isAction=true diff --git a/client/client_test.go b/client/client_test.go index 28035bd..297299d 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -2415,7 +2415,7 @@ func TestSortByConsistentTimezone(t *testing.T) { testutils.CompareGoldens(t, out, "TestSortByConsistentTimezone-query") out = captureTerminalOutput(t, tester, []string{"hishtory SPACE tquery SPACE -pipefail SPACE -tablesizing ENTER"}) out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) - testutils.CompareGoldens(t, out, "TestSortByConsistentTimezone-tquery") + testutils.CompareGoldens(t, out, fmt.Sprintf("TestSortByConsistentTimezone-tquery-isAction=%v", testutils.IsGithubAction())) } func TestZDotDir(t *testing.T) { diff --git a/client/lib/goldens/TestSortByConsistentTimezone-tquery b/client/lib/goldens/TestSortByConsistentTimezone-tquery-isAction=false similarity index 100% rename from client/lib/goldens/TestSortByConsistentTimezone-tquery rename to client/lib/goldens/TestSortByConsistentTimezone-tquery-isAction=false diff --git a/client/lib/goldens/TestSortByConsistentTimezone-tquery-isAction=true b/client/lib/goldens/TestSortByConsistentTimezone-tquery-isAction=true new file mode 100644 index 0000000..da690c8 --- /dev/null +++ b/client/lib/goldens/TestSortByConsistentTimezone-tquery-isAction=true @@ -0,0 +1,31 @@ +-pipefail -tablesizing + + + +Search Query: > -pipefail -tablesizing + +┌────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ Hostname CWD Timestamp Runtime Exit Code Command │ +│────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────│ +│ localhost /tmp/ Apr 16 2022 01:36:26 PDT 1s 2 third_entry │ +│ localhost /tmp/ Apr 16 2022 01:19:46 PDT 1s 2 second_entry │ +│ localhost /tmp/ Apr 16 2022 01:03:06 PDT 1s 2 first_entry │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +│ │ +└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +hiSHtory: Search your shell history • ctrl+h help \ No newline at end of file