Add integration to report flaky tests to datadog

client/client_test.go

@@ -13,7 +13,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/DataDog/datadog-go/statsd"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/uuid"
 	"gorm.io/gorm"
@@ -50,20 +49,6 @@ func TestMain(m *testing.M) {
 		panic(fmt.Sprintf("failed to build client: %v", err))
 	}
 
-	// Configure the integration to export test failures to datadog for better monitoring
-	if _, has_dd_api_key := os.LookupEnv("DD_API_KEY"); testutils.IsGithubAction() && has_dd_api_key {
-		ddStats, err := statsd.New("localhost:8125")
-		if err != nil {
-			err := fmt.Errorf("Failed to start DataDog statsd: %w\n", err)
-			if runtime.GOOS == "darwin" {
-				fmt.Printf("%v", err)
-			} else {
-				panic(err)
-			}
-		}
-		GLOBAL_STATSD = ddStats
-	}
-
 	// Start the tests
 	m.Run()
 }

client/test_metrics_exporter/main.go

@@ -0,0 +1,70 @@
+// Exports test metrics to DD so we can monitor for flaky tests over time
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+
+	"github.com/DataDog/datadog-go/statsd"
+	"gotest.tools/gotestsum/testjson"
+)
+
+var GLOBAL_STATSD *statsd.Client
+
+var NUM_TEST_RETRIES map[string]int
+
+func main() {
+	// Configure Datadog
+	if _, has_dd_api_key := os.LookupEnv("DD_API_KEY"); !(has_dd_api_key) {
+		fmt.Printf("Skipping exporting test stats to datadog\n")
+	}
+	ddStats, err := statsd.New("localhost:8125")
+	if err != nil {
+		err := fmt.Errorf("failed to start DataDog statsd: %w", err)
+		if runtime.GOOS == "darwin" {
+			fmt.Printf("failed init datadog: %v", err)
+			os.Exit(0)
+		} else {
+			log.Fatalf("failed to init datadog: %v", err)
+		}
+	}
+	GLOBAL_STATSD = ddStats
+
+	// Parse the test output
+	NUM_TEST_RETRIES = make(map[string]int)
+	inputFile, err := os.Open("/tmp/testrun.json")
+	if err != nil {
+		log.Fatalf("failed to open test input file: %v", err)
+	}
+	_, err = testjson.ScanTestOutput(testjson.ScanConfig{
+		Stdout:  inputFile,
+		Handler: eventHandler{},
+	})
+	if err != nil {
+		log.Fatalf("failed to scan testjson: %v", err)
+	}
+	for testId, count := range NUM_TEST_RETRIES {
+		GLOBAL_STATSD.Distribution("test_retry_count", float64(count), []string{"test:" + testId, "os:" + runtime.GOOS}, 1.0)
+	}
+}
+
+type eventHandler struct{}
+
+func (eventHandler) Event(event testjson.TestEvent, execution *testjson.Execution) error {
+	testIdentifier := fmt.Sprintf("%s#%s", event.Package, event.Test)
+	if event.Action == testjson.ActionFail {
+		GLOBAL_STATSD.Incr("test_status", []string{"result:failed", "test:" + testIdentifier, "os:" + runtime.GOOS}, 1.0)
+		NUM_TEST_RETRIES[testIdentifier] += 1
+	}
+	if event.Action == testjson.ActionPass {
+		GLOBAL_STATSD.Incr("test_status", []string{"result:passed", "test:" + testIdentifier, "os:" + runtime.GOOS}, 1.0)
+		NUM_TEST_RETRIES[testIdentifier] += 1
+	}
+	return nil
+}
+
+func (eventHandler) Err(text string) error {
+	return fmt.Errorf("unexpected error when parsing test output: %v", text)
+}

client/testutils.go

@@ -16,7 +16,6 @@ import (
 	"syscall"
 	"testing"
 
-	"github.com/DataDog/datadog-go/statsd"
 	"github.com/ddworken/hishtory/client/data"
 	"github.com/ddworken/hishtory/client/hctx"
 	"github.com/ddworken/hishtory/client/lib"
@@ -25,8 +24,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var GLOBAL_STATSD *statsd.Client
-
 type shellTester interface {
 	RunInteractiveShell(t testing.TB, script string) string
 	RunInteractiveShellRelaxed(t testing.TB, script string) (string, error)