mirror of
https://github.com/ddworken/hishtory.git
synced 2025-01-11 16:58:47 +01:00
Run integration tests in parallel to speed up testing (#175)
* Remove a few direct DB insertions to prepare for parallel tests * Revert "Remove a few direct DB insertions to prepare for parallel tests" This reverts commitf8a3552ad8. * Add rudimentary experiment of splitting tests into two chunks to make them faster * Add missing tag * Remove code that enforces that all goldens are used, since it is incompatible with how tests are currently split into chunks * Lay out the framework for checking goldens being used across all test runs * Fix missing brace * Revert "Remove code that enforces that all goldens are used, since it is incompatible with how tests are currently split into chunks" This reverts commit06cc3eedbc. * Add initial work towards checking that all goldens are used * Delete incorrect and unreferenced matrix * Upgrade actions/upload-artifact to see if that makes the download in the next job work * Alternatively, try downloading the artifact by name * Update golden checker to read all the golden artifacts * Swap to using glob to enumerate all golden files, rather than hardcoding them * Remove debugging commands * Remove goldens that are actually used * Remove another golden that is actually used * Add more comprehensive support for test sharding * Fix references to test shards and increase shard count * Shard the fuzz test * Add debug prints * Mark additional tests for sharding * Fix logic error that broke test sharding * Remove debug print * Fix incorrect logic with skipping the fuzz test * Move sharding functions to testutils and add some comments * Upgrade all setup-go actions to enable caching of deps * Remove goldens that don't exist * Remove new line * Reduce delay * Correct stage name * Remove incorrect skip code from the first version of sharding * Remove unused import * Reduce number of test shards to match GitHub's limit of 5 concurrent macos jobs * Use cask for installing homebrew to speed up github actions * More cleanup for unused goldens
This commit is contained in:
David Dworken
parent
1c963bed8a
commit
33bc582d8b
2
.github/workflows/docker-compose-test.yml
vendored
2
.github/workflows/docker-compose-test.yml
vendored
@ -12,7 +12,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: 1.21
|
||||
- name: Docker Compose setup
|
||||
|
||||
37
.github/workflows/go-test.yml
vendored
37
.github/workflows/go-test.yml
vendored
@ -14,6 +14,7 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, macos-14]
|
||||
test_shard: ["0", "1", "2", "3", "4"]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@ -45,10 +46,10 @@ jobs:
|
||||
sudo scutil --set HostName ghaction-runner-hostname
|
||||
- name: MacOS Docker Setup
|
||||
if: ${{ matrix.os == 'macos-latest' || matrix.os == 'macos-14 '}}
|
||||
continue-on-error: true
|
||||
continue-on-error: true # Since colima is flaky, and a failure here only impacts our metrics
|
||||
run: |
|
||||
# Install docker so it can be used for datadog
|
||||
brew install docker
|
||||
brew install --cask docker
|
||||
colima start
|
||||
sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
|
||||
- name: Set up Datadog
|
||||
@ -69,33 +70,53 @@ jobs:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
run: |
|
||||
go install gotest.tools/gotestsum@bc98120
|
||||
make test
|
||||
NUM_TEST_SHARDS=5 CURRENT_SHARD_NUM=${{ matrix.test_shard }} make test
|
||||
- name: Extra Delay
|
||||
run: |
|
||||
|
||||
# Add an extra short delay to allow datadog to flush metrics
|
||||
sleep 300 # 5 minutes
|
||||
sleep 90
|
||||
- name: Upload test results json
|
||||
uses: actions/upload-artifact@v3
|
||||
if: success() || failure()
|
||||
with:
|
||||
name: test-results-${{ matrix.os }}.json
|
||||
name: test-results-${{ matrix.os }}-${{ matrix.test_shard }}.json
|
||||
path: /tmp/testrun.json
|
||||
- name: Upload failed test goldens
|
||||
uses: actions/upload-artifact@v3
|
||||
if: success() || failure()
|
||||
with:
|
||||
name: test-goldens-${{ matrix.os }}.zip
|
||||
name: test-goldens-${{ matrix.os }}-${{ matrix.test_shard }}.zip
|
||||
path: /tmp/test-goldens/
|
||||
- name: Upload test log
|
||||
uses: actions/upload-artifact@v3
|
||||
if: success() || failure()
|
||||
with:
|
||||
name: testlog-${{ matrix.os }}.txt
|
||||
name: testlog-${{ matrix.os }}-${{ matrix.test_shard }}.txt
|
||||
path: /tmp/test.log
|
||||
- name: Upload used goldens
|
||||
uses: actions/upload-artifact@v4
|
||||
if: success() || failure()
|
||||
with:
|
||||
name: goldens-used-${{ matrix.os }}-${{ matrix.test_shard }}
|
||||
path: /tmp/goldens-used.txt
|
||||
|
||||
# - name: Setup tmate session
|
||||
# if: ${{ failure() }}
|
||||
# uses: mxschmitt/action-tmate@v3
|
||||
# with:
|
||||
# limit-access-to-actor: true
|
||||
# limit-access-to-actor: true
|
||||
check-goldens:
|
||||
runs-on: ubuntu-latest
|
||||
needs: test
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: 1.21
|
||||
- name: Download artifact
|
||||
uses: actions/download-artifact@v4
|
||||
- name: Check all goldens were used
|
||||
run: |
|
||||
go run client/posttest/main.go check-goldens
|
||||
2
.github/workflows/pre-commit.yml
vendored
2
.github/workflows/pre-commit.yml
vendored
@ -11,7 +11,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v3
|
||||
- uses: actions/setup-go@v3
|
||||
- uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: 1.21
|
||||
- name: Install dependencies
|
||||
|
||||
2
.github/workflows/server-releaser.yml
vendored
2
.github/workflows/server-releaser.yml
vendored
@ -23,7 +23,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: 1.21
|
||||
- name: Build server binary
|
||||
|
||||
2
.github/workflows/slsa-releaser.yml
vendored
2
.github/workflows/slsa-releaser.yml
vendored
@ -182,7 +182,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: 1.21
|
||||
- uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741
|
||||
|
||||
2
Makefile
2
Makefile
@ -9,7 +9,7 @@ forcetest: ## Force running all tests without a test cache
|
||||
make test
|
||||
|
||||
test: ## Run all tests
|
||||
TZ='America/Los_Angeles' HISHTORY_TEST=1 HISHTORY_SKIP_INIT_IMPORT=1 gotestsum --packages ./... --rerun-fails=10 --rerun-fails-max-failures=30 --format testname --jsonfile /tmp/testrun.json --post-run-command "go run client/posttest/main.go" -- -p 1 -timeout 90m
|
||||
TZ='America/Los_Angeles' HISHTORY_TEST=1 HISHTORY_SKIP_INIT_IMPORT=1 gotestsum --packages ./... --rerun-fails=10 --rerun-fails-max-failures=30 --format testname --jsonfile /tmp/testrun.json --post-run-command "go run client/posttest/main.go export" -- -p 1 -timeout 90m
|
||||
|
||||
ftest: ## Run a specific test specified via `make ftest FILTER=TestParam/testTui/color`
|
||||
go clean -testcache
|
||||
|
||||
@ -62,58 +62,58 @@ func TestParam(t *testing.T) {
|
||||
shellTesters = shellTesters[:1]
|
||||
}
|
||||
for _, tester := range shellTesters {
|
||||
t.Run("testRepeatedCommandThenQuery/"+tester.ShellName(), func(t *testing.T) { testRepeatedCommandThenQuery(t, tester) })
|
||||
t.Run("testRepeatedCommandAndQuery/"+tester.ShellName(), func(t *testing.T) { testRepeatedCommandAndQuery(t, tester) })
|
||||
t.Run("testRepeatedEnableDisable/"+tester.ShellName(), func(t *testing.T) { testRepeatedEnableDisable(t, tester) })
|
||||
t.Run("testExcludeHiddenCommand/"+tester.ShellName(), func(t *testing.T) { testExcludeHiddenCommand(t, tester) })
|
||||
t.Run("testUpdate/head->release/"+tester.ShellName(), func(t *testing.T) { testUpdateFromHeadToRelease(t, tester) })
|
||||
t.Run("testUpdate/prev->release/"+tester.ShellName(), func(t *testing.T) { testUpdateFromPrevToRelease(t, tester) })
|
||||
t.Run("testUpdate/prev->release/prod/"+tester.ShellName(), func(t *testing.T) { testUpdateFromPrevToReleaseViaProd(t, tester) })
|
||||
t.Run("testUpdate/prev->current/"+tester.ShellName(), func(t *testing.T) { testUpdateFromPrevToCurrent(t, tester) })
|
||||
t.Run("testAdvancedQuery/"+tester.ShellName(), func(t *testing.T) { testAdvancedQuery(t, tester) })
|
||||
t.Run("testIntegration/"+tester.ShellName(), func(t *testing.T) { testIntegration(t, tester, Online) })
|
||||
t.Run("testIntegration/offline/"+tester.ShellName(), func(t *testing.T) { testIntegration(t, tester, Offline) })
|
||||
t.Run("testIntegrationWithNewDevice/"+tester.ShellName(), func(t *testing.T) { testIntegrationWithNewDevice(t, tester) })
|
||||
t.Run("testHishtoryBackgroundSaving/"+tester.ShellName(), func(t *testing.T) { testHishtoryBackgroundSaving(t, tester) })
|
||||
t.Run("testDisplayTable/"+tester.ShellName(), func(t *testing.T) { testDisplayTable(t, tester) })
|
||||
t.Run("testTableDisplayCwd/"+tester.ShellName(), func(t *testing.T) { testTableDisplayCwd(t, tester) })
|
||||
t.Run("testTimestampsAreReasonablyCorrect/"+tester.ShellName(), func(t *testing.T) { testTimestampsAreReasonablyCorrect(t, tester) })
|
||||
t.Run("testRequestAndReceiveDbDump/"+tester.ShellName(), func(t *testing.T) { testRequestAndReceiveDbDump(t, tester) })
|
||||
t.Run("testInstallViaPythonScript/"+tester.ShellName(), func(t *testing.T) { testInstallViaPythonScript(t, tester) })
|
||||
t.Run("testExportWithQuery/"+tester.ShellName(), func(t *testing.T) { testExportWithQuery(t, tester) })
|
||||
t.Run("testHelpCommand/"+tester.ShellName(), func(t *testing.T) { testHelpCommand(t, tester) })
|
||||
t.Run("testReuploadHistoryEntries/"+tester.ShellName(), func(t *testing.T) { testReuploadHistoryEntries(t, tester) })
|
||||
t.Run("testHishtoryOffline/"+tester.ShellName(), func(t *testing.T) { testHishtoryOffline(t, tester) })
|
||||
t.Run("testInitialHistoryImport/"+tester.ShellName(), func(t *testing.T) { testInitialHistoryImport(t, tester) })
|
||||
t.Run("testLocalRedaction/"+tester.ShellName(), func(t *testing.T) { testLocalRedaction(t, tester, Online) })
|
||||
t.Run("testLocalRedaction/offline/"+tester.ShellName(), func(t *testing.T) { testLocalRedaction(t, tester, Offline) })
|
||||
t.Run("testRemoteRedaction/"+tester.ShellName(), func(t *testing.T) { testRemoteRedaction(t, tester) })
|
||||
t.Run("testMultipleUsers/"+tester.ShellName(), func(t *testing.T) { testMultipleUsers(t, tester) })
|
||||
t.Run("testConfigGetSet/"+tester.ShellName(), func(t *testing.T) { testConfigGetSet(t, tester) })
|
||||
t.Run("testHandleUpgradedFeatures/"+tester.ShellName(), func(t *testing.T) { testHandleUpgradedFeatures(t, tester) })
|
||||
t.Run("testCustomColumns/"+tester.ShellName(), func(t *testing.T) { testCustomColumns(t, tester) })
|
||||
t.Run("testUninstall/"+tester.ShellName(), func(t *testing.T) { testUninstall(t, tester) })
|
||||
t.Run("testPresaving/"+tester.ShellName(), func(t *testing.T) { testPresaving(t, tester, tester.ShellName()) })
|
||||
t.Run("testPresavingOffline/"+tester.ShellName(), func(t *testing.T) { testPresavingOffline(t, tester) })
|
||||
t.Run("testPresavingDisabled/"+tester.ShellName(), func(t *testing.T) { testPresavingDisabled(t, tester) })
|
||||
t.Run("testControlR/online/"+tester.ShellName(), func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Online) })
|
||||
t.Run("testControlR/offline/"+tester.ShellName(), func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Offline) })
|
||||
t.Run("testTabCompletion/"+tester.ShellName(), func(t *testing.T) { testTabCompletion(t, tester, tester.ShellName()) })
|
||||
t.Run("testRepeatedCommandThenQuery/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testRepeatedCommandThenQuery(t, tester) }))
|
||||
t.Run("testRepeatedCommandAndQuery/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testRepeatedCommandAndQuery(t, tester) }))
|
||||
t.Run("testRepeatedEnableDisable/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testRepeatedEnableDisable(t, tester) }))
|
||||
t.Run("testExcludeHiddenCommand/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testExcludeHiddenCommand(t, tester) }))
|
||||
t.Run("testUpdate/head->release/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testUpdateFromHeadToRelease(t, tester) }))
|
||||
t.Run("testUpdate/prev->release/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testUpdateFromPrevToRelease(t, tester) }))
|
||||
t.Run("testUpdate/prev->release/prod/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testUpdateFromPrevToReleaseViaProd(t, tester) }))
|
||||
t.Run("testUpdate/prev->current/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testUpdateFromPrevToCurrent(t, tester) }))
|
||||
t.Run("testAdvancedQuery/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testAdvancedQuery(t, tester) }))
|
||||
t.Run("testIntegration/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testIntegration(t, tester, Online) }))
|
||||
t.Run("testIntegration/offline/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testIntegration(t, tester, Offline) }))
|
||||
t.Run("testIntegrationWithNewDevice/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testIntegrationWithNewDevice(t, tester) }))
|
||||
t.Run("testHishtoryBackgroundSaving/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testHishtoryBackgroundSaving(t, tester) }))
|
||||
t.Run("testDisplayTable/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testDisplayTable(t, tester) }))
|
||||
t.Run("testTableDisplayCwd/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testTableDisplayCwd(t, tester) }))
|
||||
t.Run("testTimestampsAreReasonablyCorrect/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testTimestampsAreReasonablyCorrect(t, tester) }))
|
||||
t.Run("testRequestAndReceiveDbDump/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testRequestAndReceiveDbDump(t, tester) }))
|
||||
t.Run("testInstallViaPythonScript/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testInstallViaPythonScript(t, tester) }))
|
||||
t.Run("testExportWithQuery/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testExportWithQuery(t, tester) }))
|
||||
t.Run("testHelpCommand/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testHelpCommand(t, tester) }))
|
||||
t.Run("testReuploadHistoryEntries/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testReuploadHistoryEntries(t, tester) }))
|
||||
t.Run("testHishtoryOffline/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testHishtoryOffline(t, tester) }))
|
||||
t.Run("testInitialHistoryImport/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testInitialHistoryImport(t, tester) }))
|
||||
t.Run("testLocalRedaction/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testLocalRedaction(t, tester, Online) }))
|
||||
t.Run("testLocalRedaction/offline/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testLocalRedaction(t, tester, Offline) }))
|
||||
t.Run("testRemoteRedaction/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testRemoteRedaction(t, tester) }))
|
||||
t.Run("testMultipleUsers/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testMultipleUsers(t, tester) }))
|
||||
t.Run("testConfigGetSet/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testConfigGetSet(t, tester) }))
|
||||
t.Run("testHandleUpgradedFeatures/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testHandleUpgradedFeatures(t, tester) }))
|
||||
t.Run("testCustomColumns/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testCustomColumns(t, tester) }))
|
||||
t.Run("testUninstall/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testUninstall(t, tester) }))
|
||||
t.Run("testPresaving/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testPresaving(t, tester, tester.ShellName()) }))
|
||||
t.Run("testPresavingOffline/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testPresavingOffline(t, tester) }))
|
||||
t.Run("testPresavingDisabled/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testPresavingDisabled(t, tester) }))
|
||||
t.Run("testControlR/online/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Online) }))
|
||||
t.Run("testControlR/offline/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Offline) }))
|
||||
t.Run("testTabCompletion/"+tester.ShellName(), wrapTestForSharding(func(t *testing.T) { testTabCompletion(t, tester, tester.ShellName()) }))
|
||||
}
|
||||
t.Run("testTabCompletion/fish", func(t *testing.T) { testTabCompletion(t, zshTester{}, "fish") })
|
||||
t.Run("testPresaving/fish", func(t *testing.T) { testPresaving(t, zshTester{}, "fish") })
|
||||
t.Run("testControlR/fish", func(t *testing.T) { testControlR(t, bashTester{}, "fish", Online) })
|
||||
t.Run("testTui/search/online", func(t *testing.T) { testTui_search(t, Online) })
|
||||
t.Run("testTui/search/offline", func(t *testing.T) { testTui_search(t, Offline) })
|
||||
t.Run("testTui/general/online", func(t *testing.T) { testTui_general(t, Online) })
|
||||
t.Run("testTui/general/offline", func(t *testing.T) { testTui_general(t, Offline) })
|
||||
t.Run("testTui/scroll", testTui_scroll)
|
||||
t.Run("testTui/resize", testTui_resize)
|
||||
t.Run("testTui/delete", testTui_delete)
|
||||
t.Run("testTui/color", testTui_color)
|
||||
t.Run("testTui/errors", testTui_errors)
|
||||
t.Run("testTui/ai", testTui_ai)
|
||||
t.Run("testTui/defaultFilter", testTui_defaultFilter)
|
||||
t.Run("testTabCompletion/fish", wrapTestForSharding(func(t *testing.T) { testTabCompletion(t, zshTester{}, "fish") }))
|
||||
t.Run("testPresaving/fish", wrapTestForSharding(func(t *testing.T) { testPresaving(t, zshTester{}, "fish") }))
|
||||
t.Run("testControlR/fish", wrapTestForSharding(func(t *testing.T) { testControlR(t, bashTester{}, "fish", Online) }))
|
||||
t.Run("testTui/search/online", wrapTestForSharding(func(t *testing.T) { testTui_search(t, Online) }))
|
||||
t.Run("testTui/search/offline", wrapTestForSharding(func(t *testing.T) { testTui_search(t, Offline) }))
|
||||
t.Run("testTui/general/online", wrapTestForSharding(func(t *testing.T) { testTui_general(t, Online) }))
|
||||
t.Run("testTui/general/offline", wrapTestForSharding(func(t *testing.T) { testTui_general(t, Offline) }))
|
||||
t.Run("testTui/scroll", wrapTestForSharding(testTui_scroll))
|
||||
t.Run("testTui/resize", wrapTestForSharding(testTui_resize))
|
||||
t.Run("testTui/delete", wrapTestForSharding(testTui_delete))
|
||||
t.Run("testTui/color", wrapTestForSharding(testTui_color))
|
||||
t.Run("testTui/errors", wrapTestForSharding(testTui_errors))
|
||||
t.Run("testTui/ai", wrapTestForSharding(testTui_ai))
|
||||
t.Run("testTui/defaultFilter", wrapTestForSharding(testTui_defaultFilter))
|
||||
|
||||
// Assert there are no leaked connections
|
||||
assertNoLeakedConnections(t)
|
||||
@ -1038,6 +1038,7 @@ echo other`)
|
||||
}
|
||||
|
||||
func TestInstallViaPythonScriptWithCustomHishtoryPath(t *testing.T) {
|
||||
markTestForSharding(t, 0)
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
defer testutils.BackupAndRestoreEnv("HISHTORY_PATH")()
|
||||
altHishtoryPath := ".other-path"
|
||||
@ -1052,6 +1053,7 @@ func TestInstallViaPythonScriptWithCustomHishtoryPath(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestInstallViaPythonScriptInOfflineMode(t *testing.T) {
|
||||
markTestForSharding(t, 1)
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
defer testutils.BackupAndRestoreEnv("HISHTORY_OFFLINE")()
|
||||
os.Setenv("HISHTORY_OFFLINE", "1")
|
||||
@ -1108,6 +1110,7 @@ func testInstallViaPythonScriptChild(t *testing.T, tester shellTester) {
|
||||
}
|
||||
|
||||
func TestInstallViaPythonScriptFromHead(t *testing.T) {
|
||||
markTestForSharding(t, 2)
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
tester := zshTester{}
|
||||
|
||||
@ -1219,6 +1222,7 @@ func testHelpCommand(t *testing.T, tester shellTester) {
|
||||
|
||||
func TestStripBashTimePrefix(t *testing.T) {
|
||||
// Setup
|
||||
markTestForSharding(t, 4)
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
tester := bashTester{}
|
||||
installHishtory(t, tester, "")
|
||||
@ -1611,6 +1615,7 @@ func testHandleUpgradedFeatures(t *testing.T, tester shellTester) {
|
||||
|
||||
func TestFish(t *testing.T) {
|
||||
// Setup
|
||||
markTestForSharding(t, 5)
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
tester := bashTester{}
|
||||
installHishtory(t, tester, "")
|
||||
@ -2570,6 +2575,7 @@ echo bar`)
|
||||
|
||||
func TestTimestampFormat(t *testing.T) {
|
||||
// Setup
|
||||
markTestForSharding(t, 6)
|
||||
tester := zshTester{}
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
userSecret := installHishtory(t, tester, "")
|
||||
@ -2607,6 +2613,7 @@ func TestTimestampFormat(t *testing.T) {
|
||||
|
||||
func TestSortByConsistentTimezone(t *testing.T) {
|
||||
// Setup
|
||||
markTestForSharding(t, 7)
|
||||
tester := zshTester{}
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
installHishtory(t, tester, "")
|
||||
@ -2646,6 +2653,7 @@ func TestSortByConsistentTimezone(t *testing.T) {
|
||||
|
||||
func TestZDotDir(t *testing.T) {
|
||||
// Setup
|
||||
markTestForSharding(t, 8)
|
||||
tester := zshTester{}
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
defer testutils.BackupAndRestoreEnv("ZDOTDIR")()
|
||||
@ -2680,6 +2688,7 @@ func TestZDotDir(t *testing.T) {
|
||||
|
||||
func TestRemoveDuplicateRows(t *testing.T) {
|
||||
// Setup
|
||||
markTestForSharding(t, 9)
|
||||
tester := zshTester{}
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
installHishtory(t, tester, "")
|
||||
@ -2732,6 +2741,7 @@ echo foo`)
|
||||
|
||||
func TestSetConfigNoCorruption(t *testing.T) {
|
||||
// Setup
|
||||
markTestForSharding(t, 10)
|
||||
tester := zshTester{}
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
installHishtory(t, tester, "")
|
||||
@ -2766,6 +2776,7 @@ func TestSetConfigNoCorruption(t *testing.T) {
|
||||
// Test that the config retrieved from the context is a reference and there are no consistency issues with it getting out of sync
|
||||
func TestCtxConfigIsReference(t *testing.T) {
|
||||
// Setup
|
||||
markTestForSharding(t, 11)
|
||||
tester := zshTester{}
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
installHishtory(t, tester, "")
|
||||
@ -2877,6 +2888,7 @@ func createSyntheticImportEntries(t testing.TB, numSyntheticEntries int) {
|
||||
|
||||
func TestImportHistory(t *testing.T) {
|
||||
// Setup
|
||||
markTestForSharding(t, 11)
|
||||
tester := bashTester{}
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
userSecret := installHishtory(t, tester, "")
|
||||
@ -2931,6 +2943,7 @@ func BenchmarkImport(b *testing.B) {
|
||||
}
|
||||
|
||||
func TestAugmentedIsOfflineError(t *testing.T) {
|
||||
markTestForSharding(t, 12)
|
||||
defer testutils.BackupAndRestore(t)()
|
||||
installHishtory(t, zshTester{}, "")
|
||||
defer testutils.BackupAndRestoreEnv("HISHTORY_SIMULATE_NETWORK_ERROR")()
|
||||
|
||||
@ -141,6 +141,12 @@ func FuzzTestMultipleUsers(f *testing.F) {
|
||||
if skipSlowTests() {
|
||||
f.Skip("skipping slow tests")
|
||||
}
|
||||
if isShardedTestRun() {
|
||||
if currentShardNumber() != 0 {
|
||||
f.Skip("Skipping sharded test")
|
||||
}
|
||||
}
|
||||
|
||||
defer testutils.RunTestServer()()
|
||||
// Format:
|
||||
// $Op = $Key;$Device|$Command\n
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
@ -19,18 +20,16 @@ var GLOBAL_STATSD *statsd.Client = nil
|
||||
|
||||
var NUM_TEST_RETRIES map[string]int
|
||||
|
||||
var UNUSED_GOLDENS []string = []string{"TestTui-Exit", "testControlR-ControlC-bash", "testControlR-ControlC-fish",
|
||||
"testControlR-ControlC-zsh", "testControlR-SelectMultiline-bash", "testControlR-SelectMultiline-fish",
|
||||
"testControlR-SelectMultiline-zsh", "testControlR-bash-Disabled", "testControlR-fish-Disabled",
|
||||
"testControlR-zsh-Disabled", "testCustomColumns-query-isAction=false", "testCustomColumns-tquery-bash",
|
||||
"testCustomColumns-tquery-zsh", "testUninstall-post-uninstall-bash",
|
||||
"testUninstall-post-uninstall-zsh", "TestTui-ColoredOutput",
|
||||
"TestTui-ColoredOutputWithCustomColorScheme", "TestTui-ColoredOutputWithSearch", "TestTui-ColoredOutputWithSearch-Highlight",
|
||||
"TestTui-DefaultColorScheme", "TestTui-ColoredOutputWithDefaultFilter"}
|
||||
var UNUSED_GOLDENS []string = []string{"testCustomColumns-query-isAction=false", "testCustomColumns-tquery-bash",
|
||||
"testCustomColumns-tquery-zsh"}
|
||||
|
||||
func main() {
|
||||
exportMetrics()
|
||||
checkGoldensUsed()
|
||||
if os.Args[1] == "export" {
|
||||
exportMetrics()
|
||||
}
|
||||
if os.Args[1] == "check-goldens" {
|
||||
checkGoldensUsed()
|
||||
}
|
||||
}
|
||||
|
||||
func checkGoldensUsed() {
|
||||
@ -39,17 +38,24 @@ func checkGoldensUsed() {
|
||||
}
|
||||
// Read the goldens that were used
|
||||
usedGoldens := make([]string, 0)
|
||||
usedGoldensFile, err := os.Open("/tmp/goldens-used.txt")
|
||||
filenames, err := filepath.Glob("*/goldens-used.txt")
|
||||
if err != nil {
|
||||
log.Fatalf("failed to open /tmp/goldens-used.txt: %v", err)
|
||||
log.Fatalf("failed to list golden files: %v", err)
|
||||
}
|
||||
defer usedGoldensFile.Close()
|
||||
scanner := bufio.NewScanner(usedGoldensFile)
|
||||
for scanner.Scan() {
|
||||
usedGoldens = append(usedGoldens, strings.TrimSpace(scanner.Text()))
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Fatalf("failed to read lines from /tmp/goldens-used.txt: %v", err)
|
||||
fmt.Printf("Found used goldens in %#v\n", filenames)
|
||||
for _, filename := range filenames {
|
||||
usedGoldensFile, err := os.Open(filename)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to open %s: %v", filename, err)
|
||||
}
|
||||
defer usedGoldensFile.Close()
|
||||
scanner := bufio.NewScanner(usedGoldensFile)
|
||||
for scanner.Scan() {
|
||||
usedGoldens = append(usedGoldens, strings.TrimSpace(scanner.Text()))
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Fatalf("failed to read lines from /tmp/goldens-used.txt: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// List all the goldens that exist
|
||||
@ -68,11 +74,6 @@ func checkGoldensUsed() {
|
||||
// It is allowlisted to not be used
|
||||
continue
|
||||
}
|
||||
if (runtime.GOOS == "darwin" && strings.Contains(goldenName, "-linux")) ||
|
||||
(runtime.GOOS == "linux" && strings.Contains(goldenName, "-darwin")) {
|
||||
// It is for another OS
|
||||
continue
|
||||
}
|
||||
unusedGoldenErr = fmt.Errorf("golden file %v was never used", goldenName)
|
||||
fmt.Println(unusedGoldenErr)
|
||||
}
|
||||
|
||||
@ -1,2 +0,0 @@
|
||||
bash-5.2$ source /Users/david/.bashrc
|
||||
bash-5.2$ ls -Slah /
|
||||
@ -1,3 +0,0 @@
|
||||
david@Davids-MacBook-Air hishtory % ls \
|
||||
-Slah \
|
||||
/
|
||||
2
client/testdata/testControlR-bash-Disabled
vendored
2
client/testdata/testControlR-bash-Disabled
vendored
@ -1,2 +0,0 @@
|
||||
bash-5.2$ source /Users/david/.bashrc
|
||||
(reverse-i-search)`':
|
||||
2
client/testdata/testControlR-zsh-Disabled
vendored
2
client/testdata/testControlR-zsh-Disabled
vendored
@ -1,2 +0,0 @@
|
||||
david@Davids-MacBook-Air hishtory %
|
||||
bck-i-search: _
|
||||
@ -366,3 +366,59 @@ func stripRequiredPrefix(t *testing.T, out, prefix string) string {
|
||||
func stripTuiCommandPrefix(t *testing.T, out string) string {
|
||||
return stripRequiredPrefix(t, out, "hishtory tquery")
|
||||
}
|
||||
|
||||
// Wrap the given test so that it can be run on Github Actions with sharding. This
|
||||
// makes it possible to run only 1/N tests on each of N github action jobs, speeding
|
||||
// up test execution through parallelization. This is necessary since the wrapped
|
||||
// integration tests rely on OS-level globals (the shell history) that can't otherwise
|
||||
// be parallelized.
|
||||
func wrapTestForSharding(test func(t *testing.T)) func(t *testing.T) {
|
||||
shardNumberAllocator += 1
|
||||
return func(t *testing.T) {
|
||||
testShardNumber := shardNumberAllocator
|
||||
markTestForSharding(t, testShardNumber)
|
||||
test(t)
|
||||
}
|
||||
}
|
||||
|
||||
var shardNumberAllocator int = 0
|
||||
|
||||
// Returns whether this is a sharded test run. false during all normal non-github action operations.
|
||||
func isShardedTestRun() bool {
|
||||
return numTestShards() != -1 && currentShardNumber() != -1
|
||||
}
|
||||
|
||||
// Get the total number of test shards
|
||||
func numTestShards() int {
|
||||
numTestShardsStr := os.Getenv("NUM_TEST_SHARDS")
|
||||
if numTestShardsStr == "" {
|
||||
return -1
|
||||
}
|
||||
numTestShards, err := strconv.Atoi(numTestShardsStr)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to parse NUM_TEST_SHARDS: %v", err))
|
||||
}
|
||||
return numTestShards
|
||||
}
|
||||
|
||||
// Get the current shard number
|
||||
func currentShardNumber() int {
|
||||
currentShardNumberStr := os.Getenv("CURRENT_SHARD_NUM")
|
||||
if currentShardNumberStr == "" {
|
||||
return -1
|
||||
}
|
||||
currentShardNumber, err := strconv.Atoi(currentShardNumberStr)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to parse CURRENT_SHARD_NUM: %v", err))
|
||||
}
|
||||
return currentShardNumber
|
||||
}
|
||||
|
||||
// Mark the given test for sharding with the given test ID number.
|
||||
func markTestForSharding(t *testing.T, testShardNumber int) {
|
||||
if isShardedTestRun() {
|
||||
if testShardNumber%numTestShards() != currentShardNumber() {
|
||||
t.Skip("Skipping sharded test")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user