diff --git a/.github/workflows/docker-compose-test.yml b/.github/workflows/docker-compose-test.yml index ca89ed3..a76f7ee 100644 --- a/.github/workflows/docker-compose-test.yml +++ b/.github/workflows/docker-compose-test.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 1.21 - name: Docker Compose setup diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 1a4a98a..b850941 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -14,6 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, macos-14] + test_shard: ["0", "1", "2", "3", "4"] fail-fast: false steps: - uses: actions/checkout@v4 @@ -45,10 +46,10 @@ jobs: sudo scutil --set HostName ghaction-runner-hostname - name: MacOS Docker Setup if: ${{ matrix.os == 'macos-latest' || matrix.os == 'macos-14 '}} - continue-on-error: true + continue-on-error: true # Since colima is flaky, and a failure here only impacts our metrics run: | # Install docker so it can be used for datadog - brew install docker + brew install --cask docker colima start sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - name: Set up Datadog @@ -69,33 +70,53 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | go install gotest.tools/gotestsum@bc98120 - make test + NUM_TEST_SHARDS=5 CURRENT_SHARD_NUM=${{ matrix.test_shard }} make test - name: Extra Delay run: | # Add an extra short delay to allow datadog to flush metrics - sleep 300 # 5 minutes + sleep 90 - name: Upload test results json uses: actions/upload-artifact@v3 if: success() || failure() with: - name: test-results-${{ matrix.os }}.json + name: test-results-${{ matrix.os }}-${{ matrix.test_shard }}.json path: /tmp/testrun.json - name: Upload failed test goldens uses: actions/upload-artifact@v3 if: success() || failure() with: - name: test-goldens-${{ matrix.os }}.zip + name: test-goldens-${{ matrix.os }}-${{ matrix.test_shard }}.zip path: /tmp/test-goldens/ - name: Upload test log uses: actions/upload-artifact@v3 if: success() || failure() with: - name: testlog-${{ matrix.os }}.txt + name: testlog-${{ matrix.os }}-${{ matrix.test_shard }}.txt path: /tmp/test.log + - name: Upload used goldens + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: goldens-used-${{ matrix.os }}-${{ matrix.test_shard }} + path: /tmp/goldens-used.txt # - name: Setup tmate session # if: ${{ failure() }} # uses: mxschmitt/action-tmate@v3 # with: - # limit-access-to-actor: true \ No newline at end of file + # limit-access-to-actor: true + check-goldens: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.21 + - name: Download artifact + uses: actions/download-artifact@v4 + - name: Check all goldens were used + run: | + go run client/posttest/main.go check-goldens \ No newline at end of file diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 93879c1..4a51646 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: 1.21 - name: Install dependencies diff --git a/.github/workflows/server-releaser.yml b/.github/workflows/server-releaser.yml index d0e52af..21955ec 100644 --- a/.github/workflows/server-releaser.yml +++ b/.github/workflows/server-releaser.yml @@ -23,7 +23,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 1.21 - name: Build server binary diff --git a/.github/workflows/slsa-releaser.yml b/.github/workflows/slsa-releaser.yml index ea3d782..625cae1 100644 --- a/.github/workflows/slsa-releaser.yml +++ b/.github/workflows/slsa-releaser.yml @@ -182,7 +182,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 1.21 - uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741 diff --git a/Makefile b/Makefile index 6b51d76..d78f18b 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ forcetest: ## Force running all tests without a test cache make test test: ## Run all tests - TZ='America/Los_Angeles' HISHTORY_TEST=1 HISHTORY_SKIP_INIT_IMPORT=1 gotestsum --packages ./... --rerun-fails=10 --rerun-fails-max-failures=30 --format testname --jsonfile /tmp/testrun.json --post-run-command "go run client/posttest/main.go" -- -p 1 -timeout 90m + TZ='America/Los_Angeles' HISHTORY_TEST=1 HISHTORY_SKIP_INIT_IMPORT=1 gotestsum --packages ./... --rerun-fails=10 --rerun-fails-max-failures=30 --format testname --jsonfile /tmp/testrun.json --post-run-command "go run client/posttest/main.go export" -- -p 1 -timeout 90m ftest: ## Run a specific test specified via `make ftest FILTER=TestParam/testTui/color` go clean -testcache diff --git a/client/client_test.go b/client/client_test.go index 7aec368..adc9dd6 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -62,58 +62,58 @@ func TestParam(t *testing.T) { shellTesters = shellTesters[:1] } for _, tester := range shellTesters { - t.Run("testRepeatedCommandThenQuery/"+tester.ShellName(), func(t *testing.T) { testRepeatedCommandThenQuery(t, tester) }) - t.Run("testRepeatedCommandAndQuery/"+tester.ShellName(), func(t *testing.T) { testRepeatedCommandAndQuery(t, tester) }) - t.Run("testRepeatedEnableDisable/"+tester.ShellName(), func(t *testing.T) { testRepeatedEnableDisable(t, tester) }) - t.Run("testExcludeHiddenCommand/"+tester.ShellName(), func(t *testing.T) { testExcludeHiddenCommand(t, tester) }) - t.Run("testUpdate/head->release/"+tester.ShellName(), func(t *testing.T) { testUpdateFromHeadToRelease(t, tester) }) - t.Run("testUpdate/prev->release/"+tester.ShellName(), func(t *testing.T) { testUpdateFromPrevToRelease(t, tester) }) - t.Run("testUpdate/prev->release/prod/"+tester.ShellName(), func(t *testing.T) { testUpdateFromPrevToReleaseViaProd(t, tester) }) - t.Run("testUpdate/prev->current/"+tester.ShellName(), func(t *testing.T) { testUpdateFromPrevToCurrent(t, tester) }) - t.Run("testAdvancedQuery/"+tester.ShellName(), func(t *testing.T) { testAdvancedQuery(t, tester) }) - t.Run("testIntegration/"+tester.ShellName(), func(t *testing.T) { testIntegration(t, tester, Online) }) - t.Run("testIntegration/offline/"+tester.ShellName(), func(t *testing.T) { testIntegration(t, tester, Offline) }) - t.Run("testIntegrationWithNewDevice/"+tester.ShellName(), func(t *testing.T) { testIntegrationWithNewDevice(t, tester) }) - t.Run("testHishtoryBackgroundSaving/"+tester.ShellName(), func(t *testing.T) { testHishtoryBackgroundSaving(t, tester) }) - t.Run("testDisplayTable/"+tester.ShellName(), func(t *testing.T) { testDisplayTable(t, tester) }) - t.Run("testTableDisplayCwd/"+tester.ShellName(), func(t *testing.T) { testTableDisplayCwd(t, tester) }) - t.Run("testTimestampsAreReasonablyCorrect/"+tester.ShellName(), func(t *testing.T) { testTimestampsAreReasonablyCorrect(t, tester) }) - t.Run("testRequestAndReceiveDbDump/"+tester.ShellName(), func(t *testing.T) { testRequestAndReceiveDbDump(t, tester) }) - t.Run("testInstallViaPythonScript/"+tester.ShellName(), func(t *testing.T) { testInstallViaPythonScript(t, tester) }) - t.Run("testExportWithQuery/"+tester.ShellName(), func(t *testing.T) { testExportWithQuery(t, tester) }) - t.Run("testHelpCommand/"+tester.ShellName(), func(t *testing.T) { testHelpCommand(t, tester) }) - t.Run("testReuploadHistoryEntries/"+tester.ShellName(), func(t *testing.T) { testReuploadHistoryEntries(t, tester) }) - t.Run("testHishtoryOffline/"+tester.ShellName(), func(t *testing.T) { testHishtoryOffline(t, tester) }) - t.Run("testInitialHistoryImport/"+tester.ShellName(), func(t *testing.T) { testInitialHistoryImport(t, tester) }) - t.Run("testLocalRedaction/"+tester.ShellName(), func(t *testing.T) { testLocalRedaction(t, tester, Online) }) - t.Run("testLocalRedaction/offline/"+tester.ShellName(), func(t *testing.T) { testLocalRedaction(t, tester, Offline) }) - t.Run("testRemoteRedaction/"+tester.ShellName(), func(t *testing.T) { testRemoteRedaction(t, tester) }) - t.Run("testMultipleUsers/"+tester.ShellName(), func(t *testing.T) { testMultipleUsers(t, tester) }) - t.Run("testConfigGetSet/"+tester.ShellName(), func(t *testing.T) { testConfigGetSet(t, tester) }) - t.Run("testHandleUpgradedFeatures/"+tester.ShellName(), func(t *testing.T) { testHandleUpgradedFeatures(t, tester) }) - t.Run("testCustomColumns/"+tester.ShellName(), func(t *testing.T) { testCustomColumns(t, tester) }) - t.Run("testUninstall/"+tester.ShellName(), func(t *testing.T) { testUninstall(t, tester) }) - t.Run("testPresaving/"+tester.ShellName(), func(t *testing.T) { testPresaving(t, tester, tester.ShellName()) }) - t.Run("testPresavingOffline/"+tester.ShellName(), func(t *testing.T) { testPresavingOffline(t, tester) }) - t.Run("testPresavingDisabled/"+tester.ShellName(), func(t *testing.T) { testPresavingDisabled(t, tester) }) - t.Run("testControlR/online/"+tester.ShellName(), func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Online) }) - t.Run("testControlR/offline/"+tester.ShellName(), func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Offline) }) - t.Run("testTabCompletion/"+tester.ShellName(), func(t *testing.T) { testTabCompletion(t, tester, tester.ShellName()) }) + t.Run("testRepeatedCommandThenQuery/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testRepeatedCommandThenQuery(t, tester) })) + t.Run("testRepeatedCommandAndQuery/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testRepeatedCommandAndQuery(t, tester) })) + t.Run("testRepeatedEnableDisable/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testRepeatedEnableDisable(t, tester) })) + t.Run("testExcludeHiddenCommand/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testExcludeHiddenCommand(t, tester) })) + t.Run("testUpdate/head->release/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testUpdateFromHeadToRelease(t, tester) })) + t.Run("testUpdate/prev->release/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testUpdateFromPrevToRelease(t, tester) })) + t.Run("testUpdate/prev->release/prod/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testUpdateFromPrevToReleaseViaProd(t, tester) })) + t.Run("testUpdate/prev->current/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testUpdateFromPrevToCurrent(t, tester) })) + t.Run("testAdvancedQuery/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testAdvancedQuery(t, tester) })) + t.Run("testIntegration/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testIntegration(t, tester, Online) })) + t.Run("testIntegration/offline/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testIntegration(t, tester, Offline) })) + t.Run("testIntegrationWithNewDevice/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testIntegrationWithNewDevice(t, tester) })) + t.Run("testHishtoryBackgroundSaving/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testHishtoryBackgroundSaving(t, tester) })) + t.Run("testDisplayTable/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testDisplayTable(t, tester) })) + t.Run("testTableDisplayCwd/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testTableDisplayCwd(t, tester) })) + t.Run("testTimestampsAreReasonablyCorrect/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testTimestampsAreReasonablyCorrect(t, tester) })) + t.Run("testRequestAndReceiveDbDump/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testRequestAndReceiveDbDump(t, tester) })) + t.Run("testInstallViaPythonScript/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testInstallViaPythonScript(t, tester) })) + t.Run("testExportWithQuery/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testExportWithQuery(t, tester) })) + t.Run("testHelpCommand/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testHelpCommand(t, tester) })) + t.Run("testReuploadHistoryEntries/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testReuploadHistoryEntries(t, tester) })) + t.Run("testHishtoryOffline/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testHishtoryOffline(t, tester) })) + t.Run("testInitialHistoryImport/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testInitialHistoryImport(t, tester) })) + t.Run("testLocalRedaction/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testLocalRedaction(t, tester, Online) })) + t.Run("testLocalRedaction/offline/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testLocalRedaction(t, tester, Offline) })) + t.Run("testRemoteRedaction/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testRemoteRedaction(t, tester) })) + t.Run("testMultipleUsers/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testMultipleUsers(t, tester) })) + t.Run("testConfigGetSet/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testConfigGetSet(t, tester) })) + t.Run("testHandleUpgradedFeatures/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testHandleUpgradedFeatures(t, tester) })) + t.Run("testCustomColumns/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testCustomColumns(t, tester) })) + t.Run("testUninstall/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testUninstall(t, tester) })) + t.Run("testPresaving/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testPresaving(t, tester, tester.ShellName()) })) + t.Run("testPresavingOffline/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testPresavingOffline(t, tester) })) + t.Run("testPresavingDisabled/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testPresavingDisabled(t, tester) })) + t.Run("testControlR/online/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Online) })) + t.Run("testControlR/offline/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Offline) })) + t.Run("testTabCompletion/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testTabCompletion(t, tester, tester.ShellName()) })) } - t.Run("testTabCompletion/fish", func(t *testing.T) { testTabCompletion(t, zshTester{}, "fish") }) - t.Run("testPresaving/fish", func(t *testing.T) { testPresaving(t, zshTester{}, "fish") }) - t.Run("testControlR/fish", func(t *testing.T) { testControlR(t, bashTester{}, "fish", Online) }) - t.Run("testTui/search/online", func(t *testing.T) { testTui_search(t, Online) }) - t.Run("testTui/search/offline", func(t *testing.T) { testTui_search(t, Offline) }) - t.Run("testTui/general/online", func(t *testing.T) { testTui_general(t, Online) }) - t.Run("testTui/general/offline", func(t *testing.T) { testTui_general(t, Offline) }) - t.Run("testTui/scroll", testTui_scroll) - t.Run("testTui/resize", testTui_resize) - t.Run("testTui/delete", testTui_delete) - t.Run("testTui/color", testTui_color) - t.Run("testTui/errors", testTui_errors) - t.Run("testTui/ai", testTui_ai) - t.Run("testTui/defaultFilter", testTui_defaultFilter) + t.Run("testTabCompletion/fish", wrapTestForSharding(func(t *testing.T) { testTabCompletion(t, zshTester{}, "fish") })) + t.Run("testPresaving/fish", wrapTestForSharding(func(t *testing.T) { testPresaving(t, zshTester{}, "fish") })) + t.Run("testControlR/fish", wrapTestForSharding(func(t *testing.T) { testControlR(t, bashTester{}, "fish", Online) })) + t.Run("testTui/search/online", wrapTestForSharding(func(t *testing.T) { testTui_search(t, Online) })) + t.Run("testTui/search/offline", wrapTestForSharding(func(t *testing.T) { testTui_search(t, Offline) })) + t.Run("testTui/general/online", wrapTestForSharding(func(t *testing.T) { testTui_general(t, Online) })) + t.Run("testTui/general/offline", wrapTestForSharding(func(t *testing.T) { testTui_general(t, Offline) })) + t.Run("testTui/scroll", wrapTestForSharding(testTui_scroll)) + t.Run("testTui/resize", wrapTestForSharding(testTui_resize)) + t.Run("testTui/delete", wrapTestForSharding(testTui_delete)) + t.Run("testTui/color", wrapTestForSharding(testTui_color)) + t.Run("testTui/errors", wrapTestForSharding(testTui_errors)) + t.Run("testTui/ai", wrapTestForSharding(testTui_ai)) + t.Run("testTui/defaultFilter", wrapTestForSharding(testTui_defaultFilter)) // Assert there are no leaked connections assertNoLeakedConnections(t) @@ -1038,6 +1038,7 @@ echo other`) } func TestInstallViaPythonScriptWithCustomHishtoryPath(t *testing.T) { + markTestForSharding(t, 0) defer testutils.BackupAndRestore(t)() defer testutils.BackupAndRestoreEnv("HISHTORY_PATH")() altHishtoryPath := ".other-path" @@ -1052,6 +1053,7 @@ func TestInstallViaPythonScriptWithCustomHishtoryPath(t *testing.T) { } func TestInstallViaPythonScriptInOfflineMode(t *testing.T) { + markTestForSharding(t, 1) defer testutils.BackupAndRestore(t)() defer testutils.BackupAndRestoreEnv("HISHTORY_OFFLINE")() os.Setenv("HISHTORY_OFFLINE", "1") @@ -1108,6 +1110,7 @@ func testInstallViaPythonScriptChild(t *testing.T, tester shellTester) { } func TestInstallViaPythonScriptFromHead(t *testing.T) { + markTestForSharding(t, 2) defer testutils.BackupAndRestore(t)() tester := zshTester{} @@ -1219,6 +1222,7 @@ func testHelpCommand(t *testing.T, tester shellTester) { func TestStripBashTimePrefix(t *testing.T) { // Setup + markTestForSharding(t, 4) defer testutils.BackupAndRestore(t)() tester := bashTester{} installHishtory(t, tester, "") @@ -1611,6 +1615,7 @@ func testHandleUpgradedFeatures(t *testing.T, tester shellTester) { func TestFish(t *testing.T) { // Setup + markTestForSharding(t, 5) defer testutils.BackupAndRestore(t)() tester := bashTester{} installHishtory(t, tester, "") @@ -2570,6 +2575,7 @@ echo bar`) func TestTimestampFormat(t *testing.T) { // Setup + markTestForSharding(t, 6) tester := zshTester{} defer testutils.BackupAndRestore(t)() userSecret := installHishtory(t, tester, "") @@ -2607,6 +2613,7 @@ func TestTimestampFormat(t *testing.T) { func TestSortByConsistentTimezone(t *testing.T) { // Setup + markTestForSharding(t, 7) tester := zshTester{} defer testutils.BackupAndRestore(t)() installHishtory(t, tester, "") @@ -2646,6 +2653,7 @@ func TestSortByConsistentTimezone(t *testing.T) { func TestZDotDir(t *testing.T) { // Setup + markTestForSharding(t, 8) tester := zshTester{} defer testutils.BackupAndRestore(t)() defer testutils.BackupAndRestoreEnv("ZDOTDIR")() @@ -2680,6 +2688,7 @@ func TestZDotDir(t *testing.T) { func TestRemoveDuplicateRows(t *testing.T) { // Setup + markTestForSharding(t, 9) tester := zshTester{} defer testutils.BackupAndRestore(t)() installHishtory(t, tester, "") @@ -2732,6 +2741,7 @@ echo foo`) func TestSetConfigNoCorruption(t *testing.T) { // Setup + markTestForSharding(t, 10) tester := zshTester{} defer testutils.BackupAndRestore(t)() installHishtory(t, tester, "") @@ -2766,6 +2776,7 @@ func TestSetConfigNoCorruption(t *testing.T) { // Test that the config retrieved from the context is a reference and there are no consistency issues with it getting out of sync func TestCtxConfigIsReference(t *testing.T) { // Setup + markTestForSharding(t, 11) tester := zshTester{} defer testutils.BackupAndRestore(t)() installHishtory(t, tester, "") @@ -2877,6 +2888,7 @@ func createSyntheticImportEntries(t testing.TB, numSyntheticEntries int) { func TestImportHistory(t *testing.T) { // Setup + markTestForSharding(t, 11) tester := bashTester{} defer testutils.BackupAndRestore(t)() userSecret := installHishtory(t, tester, "") @@ -2931,6 +2943,7 @@ func BenchmarkImport(b *testing.B) { } func TestAugmentedIsOfflineError(t *testing.T) { + markTestForSharding(t, 12) defer testutils.BackupAndRestore(t)() installHishtory(t, zshTester{}, "") defer testutils.BackupAndRestoreEnv("HISHTORY_SIMULATE_NETWORK_ERROR")() diff --git a/client/fuzz_test.go b/client/fuzz_test.go index 3cb01b0..ab1b59e 100644 --- a/client/fuzz_test.go +++ b/client/fuzz_test.go @@ -141,6 +141,12 @@ func FuzzTestMultipleUsers(f *testing.F) { if skipSlowTests() { f.Skip("skipping slow tests") } + if isShardedTestRun() { + if currentShardNumber() != 0 { + f.Skip("Skipping sharded test") + } + } + defer testutils.RunTestServer()() // Format: // $Op = $Key;$Device|$Command\n diff --git a/client/posttest/main.go b/client/posttest/main.go index 93c7b93..a118e68 100644 --- a/client/posttest/main.go +++ b/client/posttest/main.go @@ -7,6 +7,7 @@ import ( "log" "os" "path" + "path/filepath" "runtime" "slices" "strings" @@ -19,18 +20,16 @@ var GLOBAL_STATSD *statsd.Client = nil var NUM_TEST_RETRIES map[string]int -var UNUSED_GOLDENS []string = []string{"TestTui-Exit", "testControlR-ControlC-bash", "testControlR-ControlC-fish", - "testControlR-ControlC-zsh", "testControlR-SelectMultiline-bash", "testControlR-SelectMultiline-fish", - "testControlR-SelectMultiline-zsh", "testControlR-bash-Disabled", "testControlR-fish-Disabled", - "testControlR-zsh-Disabled", "testCustomColumns-query-isAction=false", "testCustomColumns-tquery-bash", - "testCustomColumns-tquery-zsh", "testUninstall-post-uninstall-bash", - "testUninstall-post-uninstall-zsh", "TestTui-ColoredOutput", - "TestTui-ColoredOutputWithCustomColorScheme", "TestTui-ColoredOutputWithSearch", "TestTui-ColoredOutputWithSearch-Highlight", - "TestTui-DefaultColorScheme", "TestTui-ColoredOutputWithDefaultFilter"} +var UNUSED_GOLDENS []string = []string{"testCustomColumns-query-isAction=false", "testCustomColumns-tquery-bash", + "testCustomColumns-tquery-zsh"} func main() { - exportMetrics() - checkGoldensUsed() + if os.Args[1] == "export" { + exportMetrics() + } + if os.Args[1] == "check-goldens" { + checkGoldensUsed() + } } func checkGoldensUsed() { @@ -39,17 +38,24 @@ func checkGoldensUsed() { } // Read the goldens that were used usedGoldens := make([]string, 0) - usedGoldensFile, err := os.Open("/tmp/goldens-used.txt") + filenames, err := filepath.Glob("*/goldens-used.txt") if err != nil { - log.Fatalf("failed to open /tmp/goldens-used.txt: %v", err) + log.Fatalf("failed to list golden files: %v", err) } - defer usedGoldensFile.Close() - scanner := bufio.NewScanner(usedGoldensFile) - for scanner.Scan() { - usedGoldens = append(usedGoldens, strings.TrimSpace(scanner.Text())) - } - if err := scanner.Err(); err != nil { - log.Fatalf("failed to read lines from /tmp/goldens-used.txt: %v", err) + fmt.Printf("Found used goldens in %#v\n", filenames) + for _, filename := range filenames { + usedGoldensFile, err := os.Open(filename) + if err != nil { + log.Fatalf("failed to open %s: %v", filename, err) + } + defer usedGoldensFile.Close() + scanner := bufio.NewScanner(usedGoldensFile) + for scanner.Scan() { + usedGoldens = append(usedGoldens, strings.TrimSpace(scanner.Text())) + } + if err := scanner.Err(); err != nil { + log.Fatalf("failed to read lines from /tmp/goldens-used.txt: %v", err) + } } // List all the goldens that exist @@ -68,11 +74,6 @@ func checkGoldensUsed() { // It is allowlisted to not be used continue } - if (runtime.GOOS == "darwin" && strings.Contains(goldenName, "-linux")) || - (runtime.GOOS == "linux" && strings.Contains(goldenName, "-darwin")) { - // It is for another OS - continue - } unusedGoldenErr = fmt.Errorf("golden file %v was never used", goldenName) fmt.Println(unusedGoldenErr) } diff --git a/client/testdata/testControlR-SelectMultiline-bash b/client/testdata/testControlR-SelectMultiline-bash deleted file mode 100644 index 0dd1add..0000000 --- a/client/testdata/testControlR-SelectMultiline-bash +++ /dev/null @@ -1,2 +0,0 @@ -bash-5.2$ source /Users/david/.bashrc -bash-5.2$ ls -Slah / \ No newline at end of file diff --git a/client/testdata/testControlR-SelectMultiline-zsh b/client/testdata/testControlR-SelectMultiline-zsh deleted file mode 100644 index 9752a55..0000000 --- a/client/testdata/testControlR-SelectMultiline-zsh +++ /dev/null @@ -1,3 +0,0 @@ -david@Davids-MacBook-Air hishtory % ls \ --Slah \ -/ \ No newline at end of file diff --git a/client/testdata/testControlR-bash-Disabled b/client/testdata/testControlR-bash-Disabled deleted file mode 100644 index 0858bb2..0000000 --- a/client/testdata/testControlR-bash-Disabled +++ /dev/null @@ -1,2 +0,0 @@ -bash-5.2$ source /Users/david/.bashrc -(reverse-i-search)`': \ No newline at end of file diff --git a/client/testdata/testControlR-zsh-Disabled b/client/testdata/testControlR-zsh-Disabled deleted file mode 100644 index 16c8700..0000000 --- a/client/testdata/testControlR-zsh-Disabled +++ /dev/null @@ -1,2 +0,0 @@ -david@Davids-MacBook-Air hishtory % -bck-i-search: _ \ No newline at end of file diff --git a/client/testutils.go b/client/testutils.go index be30435..5016417 100644 --- a/client/testutils.go +++ b/client/testutils.go @@ -366,3 +366,59 @@ func stripRequiredPrefix(t *testing.T, out, prefix string) string { func stripTuiCommandPrefix(t *testing.T, out string) string { return stripRequiredPrefix(t, out, "hishtory tquery") } + +// Wrap the given test so that it can be run on Github Actions with sharding. This +// makes it possible to run only 1/N tests on each of N github action jobs, speeding +// up test execution through parallelization. This is necessary since the wrapped +// integration tests rely on OS-level globals (the shell history) that can't otherwise +// be parallelized. +func wrapTestForSharding(test func(t *testing.T)) func(t *testing.T) { + shardNumberAllocator += 1 + return func(t *testing.T) { + testShardNumber := shardNumberAllocator + markTestForSharding(t, testShardNumber) + test(t) + } +} + +var shardNumberAllocator int = 0 + +// Returns whether this is a sharded test run. false during all normal non-github action operations. +func isShardedTestRun() bool { + return numTestShards() != -1 && currentShardNumber() != -1 +} + +// Get the total number of test shards +func numTestShards() int { + numTestShardsStr := os.Getenv("NUM_TEST_SHARDS") + if numTestShardsStr == "" { + return -1 + } + numTestShards, err := strconv.Atoi(numTestShardsStr) + if err != nil { + panic(fmt.Errorf("failed to parse NUM_TEST_SHARDS: %v", err)) + } + return numTestShards +} + +// Get the current shard number +func currentShardNumber() int { + currentShardNumberStr := os.Getenv("CURRENT_SHARD_NUM") + if currentShardNumberStr == "" { + return -1 + } + currentShardNumber, err := strconv.Atoi(currentShardNumberStr) + if err != nil { + panic(fmt.Errorf("failed to parse CURRENT_SHARD_NUM: %v", err)) + } + return currentShardNumber +} + +// Mark the given test for sharding with the given test ID number. +func markTestForSharding(t *testing.T, testShardNumber int) { + if isShardedTestRun() { + if testShardNumber%numTestShards() != currentShardNumber() { + t.Skip("Skipping sharded test") + } + } +}