feat(metrics): Add more metrics (#278)

* add gatus_results_success and gatus_results_duration_seconds

* add metrics namespace

* add result http metrics

* add more metrics

* update

* extract endpoint type method

* initializedMetrics

* remove too many metrics

* update naming

* chore(metrics): Refactor code and merge results_dns_return_code_total, results_http_status_code_total into results_code_total

* docs(metrics): Update results_certificate_expiration_seconds description

* add TestEndpoint_Type

* remove name in table test

Co-authored-by: TwiN <twin@linux.com>

core/endpoint.go

@@ -18,6 +18,8 @@ import (
 	"github.com/TwiN/gatus/v3/util"
 )
 
+type EndpointType string
+
 const (
 	// HostHeader is the name of the header used to specify the host
 	HostHeader = "Host"
@@ -30,6 +32,14 @@ const (
 
 	// GatusUserAgent is the default user agent that Gatus uses to send requests.
 	GatusUserAgent = "Gatus/1.0"
+
+	// EndpointType enum for the endpoint type.
+	EndpointTypeDNS      EndpointType = "DNS"
+	EndpointTypeTCP      EndpointType = "TCP"
+	EndpointTypeICMP     EndpointType = "ICMP"
+	EndpointTypeSTARTTLS EndpointType = "STARTTLS"
+	EndpointTypeTLS      EndpointType = "TLS"
+	EndpointTypeHTTP     EndpointType = "HTTP"
 )
 
 var (
@@ -105,6 +115,24 @@ func (endpoint Endpoint) IsEnabled() bool {
 	return *endpoint.Enabled
 }
 
+// Type returns the endpoint type
+func (endpoint Endpoint) Type() EndpointType {
+	switch {
+	case endpoint.DNS != nil:
+		return EndpointTypeDNS
+	case strings.HasPrefix(endpoint.URL, "tcp://"):
+		return EndpointTypeTCP
+	case strings.HasPrefix(endpoint.URL, "icmp://"):
+		return EndpointTypeICMP
+	case strings.HasPrefix(endpoint.URL, "starttls://"):
+		return EndpointTypeSTARTTLS
+	case strings.HasPrefix(endpoint.URL, "tls://"):
+		return EndpointTypeTLS
+	default:
+		return EndpointTypeHTTP
+	}
+}
+
 // ValidateAndSetDefaults validates the endpoint's configuration and sets the default value of fields that have one
 func (endpoint *Endpoint) ValidateAndSetDefaults() error {
 	// Set default values
@@ -229,21 +257,16 @@ func (endpoint *Endpoint) call(result *Result) {
 	var response *http.Response
 	var err error
 	var certificate *x509.Certificate
-	isTypeDNS := endpoint.DNS != nil
+	endpointType := endpoint.Type()
-	isTypeTCP := strings.HasPrefix(endpoint.URL, "tcp://")
+	if endpointType == EndpointTypeHTTP {
-	isTypeICMP := strings.HasPrefix(endpoint.URL, "icmp://")
-	isTypeSTARTTLS := strings.HasPrefix(endpoint.URL, "starttls://")
-	isTypeTLS := strings.HasPrefix(endpoint.URL, "tls://")
-	isTypeHTTP := !isTypeDNS && !isTypeTCP && !isTypeICMP && !isTypeSTARTTLS && !isTypeTLS
-	if isTypeHTTP {
 		request = endpoint.buildHTTPRequest()
 	}
 	startTime := time.Now()
-	if isTypeDNS {
+	if endpointType == EndpointTypeDNS {
 		endpoint.DNS.query(endpoint.URL, result)
 		result.Duration = time.Since(startTime)
-	} else if isTypeSTARTTLS || isTypeTLS {
+	} else if endpointType == EndpointTypeSTARTTLS || endpointType == EndpointTypeTLS {
-		if isTypeSTARTTLS {
+		if endpointType == EndpointTypeSTARTTLS {
 			result.Connected, certificate, err = client.CanPerformStartTLS(strings.TrimPrefix(endpoint.URL, "starttls://"), endpoint.ClientConfig)
 		} else {
 			result.Connected, certificate, err = client.CanPerformTLS(strings.TrimPrefix(endpoint.URL, "tls://"), endpoint.ClientConfig)
@@ -254,10 +277,10 @@ func (endpoint *Endpoint) call(result *Result) {
 		}
 		result.Duration = time.Since(startTime)
 		result.CertificateExpiration = time.Until(certificate.NotAfter)
-	} else if isTypeTCP {
+	} else if endpointType == EndpointTypeTCP {
 		result.Connected = client.CanCreateTCPConnection(strings.TrimPrefix(endpoint.URL, "tcp://"), endpoint.ClientConfig)
 		result.Duration = time.Since(startTime)
-	} else if isTypeICMP {
+	} else if endpointType == EndpointTypeICMP {
 		result.Connected, result.Duration = client.Ping(strings.TrimPrefix(endpoint.URL, "icmp://"), endpoint.ClientConfig)
 	} else {
 		response, err = client.GetHTTPClient(endpoint.ClientConfig).Do(request)

core/endpoint_test.go

@@ -23,6 +23,62 @@ func TestEndpoint_IsEnabled(t *testing.T) {
 	}
 }
 
+func TestEndpoint_Type(t *testing.T) {
+	type fields struct {
+		URL string
+		DNS *DNS
+	}
+	tests := []struct {
+		fields fields
+		want   EndpointType
+	}{{
+		fields: fields{
+			URL: "8.8.8.8",
+			DNS: &DNS{
+				QueryType: "A",
+				QueryName: "example.com",
+			},
+		},
+		want: EndpointTypeDNS,
+	}, {
+		fields: fields{
+			URL: "tcp://127.0.0.1:6379",
+		},
+		want: EndpointTypeTCP,
+	}, {
+		fields: fields{
+			URL: "icmp://example.com",
+		},
+		want: EndpointTypeICMP,
+	}, {
+		fields: fields{
+			URL: "starttls://smtp.gmail.com:587",
+		},
+		want: EndpointTypeSTARTTLS,
+	}, {
+		fields: fields{
+			URL: "tls://example.com:443",
+		},
+		want: EndpointTypeTLS,
+	}, {
+		fields: fields{
+			URL: "https://twin.sh/health",
+		},
+		want: EndpointTypeHTTP,
+	}}
+	for _, tt := range tests {
+		t.Run(string(tt.want), func(t *testing.T) {
+			endpoint := Endpoint{
+				URL: tt.fields.URL,
+				DNS: tt.fields.DNS,
+			}
+			if got := endpoint.Type(); got != tt.want {
+				t.Errorf("Endpoint.Type() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 func TestEndpoint_ValidateAndSetDefaults(t *testing.T) {
 	condition := Condition("[STATUS] == 200")
 	endpoint := Endpoint{

metric/metric.go

@@ -8,20 +8,66 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
+const namespace = "gatus" // The prefix of the metrics
+
 var (
-	// This will be initialized once PublishMetricsForEndpoint.
+	initializedMetrics bool // Whether the metrics have been initialized
-	// The reason why we're doing this is that if metrics are disabled, we don't want to initialize it unnecessarily.
+
-	resultCount *prometheus.CounterVec = nil
+	resultTotal                        *prometheus.CounterVec
+	resultDurationSeconds              *prometheus.GaugeVec
+	resultConnectedTotal               *prometheus.CounterVec
+	resultCodeTotal                    *prometheus.CounterVec
+	resultCertificateExpirationSeconds *prometheus.GaugeVec
 )
 
+func initializePrometheusMetrics() {
+	resultTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: namespace,
+		Name:      "results_total",
+		Help:      "Number of results per endpoint",
+	}, []string{"key", "group", "name", "type", "success"})
+	resultDurationSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Name:      "results_duration_seconds",
+		Help:      "Duration of the request in seconds",
+	}, []string{"key", "group", "name", "type"})
+	resultConnectedTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: namespace,
+		Name:      "results_connected_total",
+		Help:      "Total number of results in which a connection was successfully established",
+	}, []string{"key", "group", "name", "type"})
+	resultCodeTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: namespace,
+		Name:      "results_code_total",
+		Help:      "Total number of results by code",
+	}, []string{"key", "group", "name", "type", "code"})
+	resultCertificateExpirationSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Name:      "results_certificate_expiration_seconds",
+		Help:      "Number of seconds until the certificate expires",
+	}, []string{"key", "group", "name", "type"})
+}
+
 // PublishMetricsForEndpoint publishes metrics for the given endpoint and its result.
 // These metrics will be exposed at /metrics if the metrics are enabled
 func PublishMetricsForEndpoint(endpoint *core.Endpoint, result *core.Result) {
-	if resultCount == nil {
+	if !initializedMetrics {
-		resultCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		initializePrometheusMetrics()
-			Name: "gatus_results_total",
+		initializedMetrics = true
-			Help: "Number of results per endpoint",
+	}
-		}, []string{"key", "group", "name", "success"})
+	endpointType := endpoint.Type()
+	resultTotal.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType), strconv.FormatBool(result.Success)).Inc()
+	resultDurationSeconds.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType)).Set(result.Duration.Seconds())
+	if result.Connected {
+		resultConnectedTotal.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType)).Inc()
+	}
+	if result.DNSRCode != "" {
+		resultCodeTotal.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType), result.DNSRCode).Inc()
+	}
+	if result.HTTPStatus != 0 {
+		resultCodeTotal.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType), strconv.Itoa(result.HTTPStatus)).Inc()
+	}
+	if result.CertificateExpiration != 0 {
+		resultCertificateExpirationSeconds.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, string(endpointType)).Set(result.CertificateExpiration.Seconds())
 	}
-	resultCount.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, strconv.FormatBool(result.Success)).Inc()
 }