Run integration tests in parallel to speed up testing (#175)

* Remove a few direct DB insertions to prepare for parallel tests

* Revert "Remove a few direct DB insertions to prepare for parallel tests"

This reverts commit f8a3552ad8.

* Add rudimentary experiment of splitting tests into two chunks to make them faster

* Add missing tag

* Remove code that enforces that all goldens are used, since it is incompatible with how tests are currently split into chunks

* Lay out the framework for checking goldens being used across all test runs

* Fix missing brace

* Revert "Remove code that enforces that all goldens are used, since it is incompatible with how tests are currently split into chunks"

This reverts commit 06cc3eedbc.

* Add initial work towards checking that all goldens are used

* Delete incorrect and unreferenced matrix

* Upgrade actions/upload-artifact to see if that makes the download in the next job work

* Alternatively, try downloading the artifact by name

* Update golden checker to read all the golden artifacts

* Swap to using glob to enumerate all golden files, rather than hardcoding them

* Remove debugging commands

* Remove goldens that are actually used

* Remove another golden that is actually used

* Add more comprehensive support for test sharding

* Fix references to test shards and increase shard count

* Shard the fuzz test

* Add debug prints

* Mark additional tests for sharding

* Fix logic error that broke test sharding

* Remove debug print

* Fix incorrect logic with skipping the fuzz test

* Move sharding functions to testutils and add some comments

* Upgrade all setup-go actions to enable caching of deps

* Remove goldens that don't exist

* Remove new line

* Reduce delay

* Correct stage name

* Remove incorrect skip code from the first version of sharding

* Remove unused import

* Reduce number of test shards to match GitHub's limit of 5 concurrent macos jobs

* Use cask for installing homebrew to speed up github actions

* More cleanup for unused goldens

client/testutils.go

@@ -366,3 +366,59 @@ func stripRequiredPrefix(t *testing.T, out, prefix string) string {
 func stripTuiCommandPrefix(t *testing.T, out string) string {
 	return stripRequiredPrefix(t, out, "hishtory tquery")
 }
+
+// Wrap the given test so that it can be run on Github Actions with sharding. This
+// makes it possible to run only 1/N tests on each of N github action jobs, speeding
+// up test execution through parallelization. This is necessary since the wrapped
+// integration tests rely on OS-level globals (the shell history) that can't otherwise
+// be parallelized.
+func wrapTestForSharding(test func(t *testing.T)) func(t *testing.T) {
+	shardNumberAllocator += 1
+	return func(t *testing.T) {
+		testShardNumber := shardNumberAllocator
+		markTestForSharding(t, testShardNumber)
+		test(t)
+	}
+}
+
+var shardNumberAllocator int = 0
+
+// Returns whether this is a sharded test run. false during all normal non-github action operations.
+func isShardedTestRun() bool {
+	return numTestShards() != -1 && currentShardNumber() != -1
+}
+
+// Get the total number of test shards
+func numTestShards() int {
+	numTestShardsStr := os.Getenv("NUM_TEST_SHARDS")
+	if numTestShardsStr == "" {
+		return -1
+	}
+	numTestShards, err := strconv.Atoi(numTestShardsStr)
+	if err != nil {
+		panic(fmt.Errorf("failed to parse NUM_TEST_SHARDS: %v", err))
+	}
+	return numTestShards
+}
+
+// Get the current shard number
+func currentShardNumber() int {
+	currentShardNumberStr := os.Getenv("CURRENT_SHARD_NUM")
+	if currentShardNumberStr == "" {
+		return -1
+	}
+	currentShardNumber, err := strconv.Atoi(currentShardNumberStr)
+	if err != nil {
+		panic(fmt.Errorf("failed to parse CURRENT_SHARD_NUM: %v", err))
+	}
+	return currentShardNumber
+}
+
+// Mark the given test for sharding with the given test ID number.
+func markTestForSharding(t *testing.T, testShardNumber int) {
+	if isShardedTestRun() {
+		if testShardNumber%numTestShards() != currentShardNumber() {
+			t.Skip("Skipping sharded test")
+		}
+	}
+}