Swap to checking golden usage post-tests so that it covers all golden usages, and not just per-package

Makefile

@@ -3,7 +3,7 @@ forcetest:
 	make test
 
 test:
-	TZ='America/Los_Angeles' HISHTORY_TEST=1 HISHTORY_SKIP_INIT_IMPORT=1 gotestsum --packages ./... --rerun-fails=10 --rerun-fails-max-failures=30 --format testname --jsonfile /tmp/testrun.json --post-run-command "go run client/test_metrics_exporter/main.go" -- -p 1 -timeout 90m  
+	TZ='America/Los_Angeles' HISHTORY_TEST=1 HISHTORY_SKIP_INIT_IMPORT=1 gotestsum --packages ./... --rerun-fails=10 --rerun-fails-max-failures=30 --format testname --jsonfile /tmp/testrun.json --post-run-command "go run client/posttest/main.go" -- -p 1 -timeout 90m  
 
 ftest:
 	go clean -testcache

client/client_test.go

@@ -53,9 +53,6 @@ func TestMain(m *testing.M) {
 
 	// Start the tests
 	m.Run()
-
-	// Teardown
-	testutils.AssertAllGoldensUsed()
 }
 
 var shellTesters []shellTester = []shellTester{bashTester{}, zshTester{}}

client/posttest/main.go

@@ -2,10 +2,14 @@
 package main
 
 import (
+	"bufio"
 	"fmt"
 	"log"
 	"os"
+	"path"
 	"runtime"
+	"slices"
+	"strings"
 
 	"github.com/DataDog/datadog-go/statsd"
 	"gotest.tools/gotestsum/testjson"
@@ -15,7 +19,58 @@ var GLOBAL_STATSD *statsd.Client = nil
 
 var NUM_TEST_RETRIES map[string]int
 
+var UNUSED_GOLDENS []string = []string{"TestTui-Exit", "testControlR-ControlC-bash", "testControlR-ControlC-fish",
+	"testControlR-ControlC-zsh", "testControlR-SelectMultiline-bash", "testControlR-SelectMultiline-fish",
+	"testControlR-SelectMultiline-zsh", "testControlR-bash-Disabled", "testControlR-fish-Disabled",
+	"testControlR-zsh-Disabled", "testCustomColumns-query-isAction=false", "testCustomColumns-tquery-bash",
+	"testCustomColumns-tquery-zsh", "testUninstall-post-uninstall-bash",
+	"testUninstall-post-uninstall-zsh"}
+
 func main() {
+	checkGoldensUsed()
+	exportMetrics()
+}
+
+func checkGoldensUsed() {
+	if os.Getenv("HISHTORY_FILTERED_TEST") != "" {
+		return
+	}
+	// Read the goldens that were used
+	usedGoldens := make([]string, 0)
+	usedGoldensFile, err := os.Open("/tmp/goldens-used.txt")
+	if err != nil {
+		log.Fatalf("failed to open /tmp/goldens-used.txt: %v", err)
+	}
+	defer usedGoldensFile.Close()
+	scanner := bufio.NewScanner(usedGoldensFile)
+	for scanner.Scan() {
+		usedGoldens = append(usedGoldens, strings.TrimSpace(scanner.Text()))
+	}
+	if err := scanner.Err(); err != nil {
+		log.Fatalf("failed to read lines from /tmp/goldens-used.txt: %v", err)
+	}
+
+	// List all the goldens that exist
+	goldensDir := "client/testdata/"
+	files, err := os.ReadDir(goldensDir)
+	if err != nil {
+		panic(fmt.Errorf("failed to list files in %s: %w", goldensDir, err))
+	}
+
+	// And check for mismatches
+	for _, f := range files {
+		goldenName := path.Base(f.Name())
+		if !slices.Contains(usedGoldens, goldenName) && !slices.Contains(UNUSED_GOLDENS, goldenName) {
+			err = fmt.Errorf("golden file %v was never used", goldenName)
+			fmt.Println(err)
+			// TODO: Add a panic(err) here too
+		}
+	}
+	fmt.Println("Validated that all goldens in testdata/ were referenced!")
+
+}
+
+func exportMetrics() {
 	// Configure Datadog
 	if _, has_dd_api_key := os.LookupEnv("DD_API_KEY"); has_dd_api_key {
 		ddStats, err := statsd.New("localhost:8125")

shared/testutils/testutils.go

@@ -11,7 +11,6 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
-	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -28,11 +27,9 @@ const (
 )
 
 var initialWd string
-var usedGoldens map[string]bool
 
 func init() {
 	initialWd = getInitialWd()
-	usedGoldens = make(map[string]bool)
 }
 
 func getInitialWd() string {
@@ -361,36 +358,20 @@ func persistLog() {
 	checkError(err)
 }
 
-var UNUSED_GOLDENS []string = []string{"TestTui-Exit", "testControlR-ControlC-bash", "testControlR-ControlC-fish",
-	"testControlR-ControlC-zsh", "testControlR-SelectMultiline-bash", "testControlR-SelectMultiline-fish",
-	"testControlR-SelectMultiline-zsh", "testControlR-bash-Disabled", "testControlR-fish-Disabled",
-	"testControlR-zsh-Disabled", "testCustomColumns-query-isAction=false", "testCustomColumns-tquery-bash",
-	"testCustomColumns-tquery-zsh", "testUninstall-post-uninstall-bash",
-	"testUninstall-post-uninstall-zsh"}
-
-func AssertAllGoldensUsed() {
-	if os.Getenv("HISHTORY_FILTERED_TEST") != "" {
-		return
-	}
-	goldensDir := path.Join(initialWd, "client/testdata/")
-	files, err := os.ReadDir(goldensDir)
+func recordUsingGolden(t testing.TB, goldenName string) {
+	f, err := os.OpenFile("/tmp/goldens-used.txt",
+		os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 	if err != nil {
-		panic(fmt.Errorf("failed to list files in %s: %w", goldensDir, err))
+		t.Fatalf("failed to open file to record using golden: %v", err)
 	}
-	for _, f := range files {
-		goldenName := path.Base(f.Name())
-		_, present := usedGoldens[goldenName]
-		if !present && !slices.Contains(UNUSED_GOLDENS, goldenName) {
-			err = fmt.Errorf("golden file %v was never used", goldenName)
-			fmt.Println(err)
-			// TODO: Add a panic(err) here too
+	defer f.Close()
+	if _, err := f.WriteString(goldenName + "\n"); err != nil {
+		t.Fatalf("failed to append to file to record using golden: %v", err)
 	}
 }
-	fmt.Println("Validated that all goldens in testdata/ were referenced!")
-}
 
 func CompareGoldens(t testing.TB, out, goldenName string) {
-	usedGoldens[goldenName] = true
+	recordUsingGolden(t, goldenName)
 	out = normalizeHostnames(out)
 	goldenPath := path.Join(initialWd, "client/testdata/", goldenName)
 	expected, err := os.ReadFile(goldenPath)