diff --git a/Makefile b/Makefile index b372783..0ba99df 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ forcetest: make test test: - TZ='America/Los_Angeles' HISHTORY_TEST=1 HISHTORY_SKIP_INIT_IMPORT=1 gotestsum --packages ./... --rerun-fails=10 --rerun-fails-max-failures=30 --format testname --jsonfile /tmp/testrun.json --post-run-command "go run client/test_metrics_exporter/main.go" -- -p 1 -timeout 90m + TZ='America/Los_Angeles' HISHTORY_TEST=1 HISHTORY_SKIP_INIT_IMPORT=1 gotestsum --packages ./... --rerun-fails=10 --rerun-fails-max-failures=30 --format testname --jsonfile /tmp/testrun.json --post-run-command "go run client/posttest/main.go" -- -p 1 -timeout 90m ftest: go clean -testcache diff --git a/client/client_test.go b/client/client_test.go index 83574c3..0bb888c 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -53,9 +53,6 @@ func TestMain(m *testing.M) { // Start the tests m.Run() - - // Teardown - testutils.AssertAllGoldensUsed() } var shellTesters []shellTester = []shellTester{bashTester{}, zshTester{}} diff --git a/client/test_metrics_exporter/main.go b/client/posttest/main.go similarity index 58% rename from client/test_metrics_exporter/main.go rename to client/posttest/main.go index e173c5e..e785d24 100644 --- a/client/test_metrics_exporter/main.go +++ b/client/posttest/main.go @@ -2,10 +2,14 @@ package main import ( + "bufio" "fmt" "log" "os" + "path" "runtime" + "slices" + "strings" "github.com/DataDog/datadog-go/statsd" "gotest.tools/gotestsum/testjson" @@ -15,7 +19,58 @@ var GLOBAL_STATSD *statsd.Client = nil var NUM_TEST_RETRIES map[string]int +var UNUSED_GOLDENS []string = []string{"TestTui-Exit", "testControlR-ControlC-bash", "testControlR-ControlC-fish", + "testControlR-ControlC-zsh", "testControlR-SelectMultiline-bash", "testControlR-SelectMultiline-fish", + "testControlR-SelectMultiline-zsh", "testControlR-bash-Disabled", "testControlR-fish-Disabled", + "testControlR-zsh-Disabled", "testCustomColumns-query-isAction=false", "testCustomColumns-tquery-bash", + "testCustomColumns-tquery-zsh", "testUninstall-post-uninstall-bash", + "testUninstall-post-uninstall-zsh"} + func main() { + checkGoldensUsed() + exportMetrics() +} + +func checkGoldensUsed() { + if os.Getenv("HISHTORY_FILTERED_TEST") != "" { + return + } + // Read the goldens that were used + usedGoldens := make([]string, 0) + usedGoldensFile, err := os.Open("/tmp/goldens-used.txt") + if err != nil { + log.Fatalf("failed to open /tmp/goldens-used.txt: %v", err) + } + defer usedGoldensFile.Close() + scanner := bufio.NewScanner(usedGoldensFile) + for scanner.Scan() { + usedGoldens = append(usedGoldens, strings.TrimSpace(scanner.Text())) + } + if err := scanner.Err(); err != nil { + log.Fatalf("failed to read lines from /tmp/goldens-used.txt: %v", err) + } + + // List all the goldens that exist + goldensDir := "client/testdata/" + files, err := os.ReadDir(goldensDir) + if err != nil { + panic(fmt.Errorf("failed to list files in %s: %w", goldensDir, err)) + } + + // And check for mismatches + for _, f := range files { + goldenName := path.Base(f.Name()) + if !slices.Contains(usedGoldens, goldenName) && !slices.Contains(UNUSED_GOLDENS, goldenName) { + err = fmt.Errorf("golden file %v was never used", goldenName) + fmt.Println(err) + // TODO: Add a panic(err) here too + } + } + fmt.Println("Validated that all goldens in testdata/ were referenced!") + +} + +func exportMetrics() { // Configure Datadog if _, has_dd_api_key := os.LookupEnv("DD_API_KEY"); has_dd_api_key { ddStats, err := statsd.New("localhost:8125") diff --git a/shared/testutils/testutils.go b/shared/testutils/testutils.go index 17d4a03..163ce69 100644 --- a/shared/testutils/testutils.go +++ b/shared/testutils/testutils.go @@ -11,7 +11,6 @@ import ( "path" "path/filepath" "runtime" - "slices" "strings" "testing" "time" @@ -28,11 +27,9 @@ const ( ) var initialWd string -var usedGoldens map[string]bool func init() { initialWd = getInitialWd() - usedGoldens = make(map[string]bool) } func getInitialWd() string { @@ -361,36 +358,20 @@ func persistLog() { checkError(err) } -var UNUSED_GOLDENS []string = []string{"TestTui-Exit", "testControlR-ControlC-bash", "testControlR-ControlC-fish", - "testControlR-ControlC-zsh", "testControlR-SelectMultiline-bash", "testControlR-SelectMultiline-fish", - "testControlR-SelectMultiline-zsh", "testControlR-bash-Disabled", "testControlR-fish-Disabled", - "testControlR-zsh-Disabled", "testCustomColumns-query-isAction=false", "testCustomColumns-tquery-bash", - "testCustomColumns-tquery-zsh", "testUninstall-post-uninstall-bash", - "testUninstall-post-uninstall-zsh"} - -func AssertAllGoldensUsed() { - if os.Getenv("HISHTORY_FILTERED_TEST") != "" { - return - } - goldensDir := path.Join(initialWd, "client/testdata/") - files, err := os.ReadDir(goldensDir) +func recordUsingGolden(t testing.TB, goldenName string) { + f, err := os.OpenFile("/tmp/goldens-used.txt", + os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { - panic(fmt.Errorf("failed to list files in %s: %w", goldensDir, err)) + t.Fatalf("failed to open file to record using golden: %v", err) } - for _, f := range files { - goldenName := path.Base(f.Name()) - _, present := usedGoldens[goldenName] - if !present && !slices.Contains(UNUSED_GOLDENS, goldenName) { - err = fmt.Errorf("golden file %v was never used", goldenName) - fmt.Println(err) - // TODO: Add a panic(err) here too - } + defer f.Close() + if _, err := f.WriteString(goldenName + "\n"); err != nil { + t.Fatalf("failed to append to file to record using golden: %v", err) } - fmt.Println("Validated that all goldens in testdata/ were referenced!") } func CompareGoldens(t testing.TB, out, goldenName string) { - usedGoldens[goldenName] = true + recordUsingGolden(t, goldenName) out = normalizeHostnames(out) goldenPath := path.Join(initialWd, "client/testdata/", goldenName) expected, err := os.ReadFile(goldenPath)