Add datadog metric to keep track of 503 errors

backend/server/internal/server/middleware.go

@@ -108,12 +108,15 @@ func withLogging(s *statsd.Client, out io.Writer) Middleware {
 
 // withPanicGuard is the last defence from a panic. it will log them and return a 503 error
 // to the client and prevent the http server from breaking
-func withPanicGuard() Middleware {
+func withPanicGuard(s *statsd.Client) Middleware {
 	return func(h http.Handler) http.Handler {
 		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 			defer func() {
 				if r := recover(); r != nil {
 					fmt.Printf("panic: %s\n", r)
+					if s != nil {
+						s.Incr("hishtory.error", []string{"handler:" + getFunctionName(h)}, 1.0)
+					}
 					// Note that we need to return a 503 error code since that is the error handled by the client in lib.IsOfflineError
 					rw.WriteHeader(http.StatusServiceUnavailable)
 				}

backend/server/internal/server/middleware_test.go

@@ -86,7 +86,7 @@ func TestPanicGuard(t *testing.T) {
 	w := httptest.NewRecorder()
 	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Header.Add("X-Real-Ip", "127.0.0.1")
-	wrappedHandler := withPanicGuard()(handler)
+	wrappedHandler := withPanicGuard(nil)(handler)
 
 	var panicked bool
 	func() {
@@ -115,7 +115,7 @@ func TestPanicGuardNoPanic(t *testing.T) {
 	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Header.Add("X-Real-Ip", "127.0.0.1")
 
-	wrappedHandler := withPanicGuard()(handler)
+	wrappedHandler := withPanicGuard(nil)(handler)
 
 	var panicked bool
 	func() {
@@ -174,7 +174,7 @@ func TestMergeMiddlewares(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			var out strings.Builder
 			middlewares := mergeMiddlewares(
-				withPanicGuard(),
+				withPanicGuard(nil),
 				withLogging(nil, &out),
 			)
 

backend/server/internal/server/srv.go

@@ -95,7 +95,7 @@ func (s *Server) Run(ctx context.Context, addr string) error {
 		}()
 	}
 	middlewares := mergeMiddlewares(
-		withPanicGuard(),
+		withPanicGuard(s.statsd),
 		withLogging(s.statsd, os.Stdout),
 	)