diff --git a/.examples/docker-compose-grafana-prometheus/README.md b/.examples/docker-compose-grafana-prometheus/README.md new file mode 100644 index 00000000..72a2f16c --- /dev/null +++ b/.examples/docker-compose-grafana-prometheus/README.md @@ -0,0 +1,23 @@ +## Usage +```console +docker-compose up +``` +Once you've done the above, you should be able to access the Grafana dashboard at `http://localhost:3000`. + +## Queries +Gatus uses Prometheus counters. + +Total results per minute: +``` +sum(rate(gatus_results_total[5m])*60) by (key) +``` + +Total successful results per minute: +``` +sum(rate(gatus_results_total{success="true"}[5m])*60) by (key) +``` + +Total unsuccessful results per minute: +``` +sum(rate(gatus_results_total{success="true"}[5m])*60) by (key) +``` \ No newline at end of file diff --git a/.examples/docker-compose-grafana-prometheus/grafana/provisioning/dashboards/main.json b/.examples/docker-compose-grafana-prometheus/grafana/provisioning/dashboards/main.json index 4ad7bf0d..8534737c 100644 --- a/.examples/docker-compose-grafana-prometheus/grafana/provisioning/dashboards/main.json +++ b/.examples/docker-compose-grafana-prometheus/grafana/provisioning/dashboards/main.json @@ -15,7 +15,6 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 2, "links": [], "panels": [ { @@ -24,20 +23,26 @@ "dashLength": 10, "dashes": false, "datasource": null, + "description": "Number of results per minute", "fill": 1, "fillGradient": 0, "gridPos": { - "h": 14, + "h": 7, "w": 12, "x": 0, "y": 0 }, "id": 2, + "interval": "", "legend": { + "alignAsTable": false, "avg": false, "current": false, + "hideEmpty": false, + "hideZero": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false @@ -58,9 +63,13 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(gatus_tasks[30s])) by (endpoint)", + "expr": "sum(rate(gatus_results_total[5m])*60) by (key)", + "format": "time_series", + "hide": false, + "instant": false, "interval": "30s", - "legendFormat": "{{endpoint}}", + "intervalFactor": 1, + "legendFormat": "{{key}}", "refId": "A" } ], @@ -68,7 +77,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "All tasks executed over time", + "title": "Total results per minute", "tooltip": { "shared": true, "sort": 0, @@ -84,6 +93,7 @@ }, "yaxes": [ { + "decimals": null, "format": "short", "label": null, "logBase": 1, @@ -119,93 +129,6 @@ "x": 12, "y": 0 }, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(gatus_tasks{success=\"false\"}[30s])) by (endpoint)", - "interval": "30s", - "legendFormat": "{{endpoint}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Unsuccessful tasks", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 7 - }, "id": 5, "legend": { "avg": false, @@ -232,10 +155,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(gatus_tasks{success=\"true\"}[30s])) by (endpoint)", + "expr": "sum(rate(gatus_results_total{success=\"true\"}[5m])*60) by (key)", "instant": false, "interval": "30s", - "legendFormat": "{{endpoint}}", + "legendFormat": "{{key}}", "refId": "A" } ], @@ -243,7 +166,183 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Successful tasks", + "title": "Successful results per minute", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorPostfix": false, + "colorPrefix": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 7, + "interval": "", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "rate(gatus_results_total{success=\"false\"}[1m])*60", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "Unsuccessful results", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(gatus_results_total{success=\"false\"}[5m])*60) by (key)", + "interval": "30s", + "legendFormat": "{{key}} ", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Unsuccessful results per minute", "tooltip": { "shared": true, "sort": 0, @@ -289,12 +388,12 @@ "list": [] }, "time": { - "from": "now-30m", + "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Gatus", "uid": "KPI7Qj1Wk", - "version": 3 + "version": 1 } \ No newline at end of file diff --git a/metric/metric.go b/metric/metric.go index b5431adf..a3e8dfcd 100644 --- a/metric/metric.go +++ b/metric/metric.go @@ -1,9 +1,7 @@ package metric import ( - "fmt" "strconv" - "sync" "github.com/TwiN/gatus/v3/core" "github.com/prometheus/client_golang/prometheus" @@ -11,24 +9,19 @@ import ( ) var ( - gauges = map[string]*prometheus.GaugeVec{} - rwLock sync.RWMutex + // This will be initialized once PublishMetricsForEndpoint. + // The reason why we're doing this is that if metrics are disabled, we don't want to initialize it unnecessarily. + resultCount *prometheus.CounterVec = nil ) // PublishMetricsForEndpoint publishes metrics for the given endpoint and its result. // These metrics will be exposed at /metrics if the metrics are enabled func PublishMetricsForEndpoint(endpoint *core.Endpoint, result *core.Result) { - rwLock.Lock() - gauge, exists := gauges[fmt.Sprintf("%s_%s", endpoint.Name, endpoint.URL)] - if !exists { - gauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: "gatus", - Name: "tasks", - // TODO: remove the "service" key in v4.0.0, as it is only kept for backward compatibility - ConstLabels: prometheus.Labels{"service": endpoint.Name, "endpoint": endpoint.Name, "url": endpoint.URL}, - }, []string{"status", "success"}) - gauges[fmt.Sprintf("%s_%s", endpoint.Name, endpoint.URL)] = gauge + if resultCount == nil { + resultCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "gatus_results_total", + Help: "Number of results per endpoint", + }, []string{"key", "group", "name", "success"}) } - rwLock.Unlock() - gauge.WithLabelValues(strconv.Itoa(result.HTTPStatus), strconv.FormatBool(result.Success)).Inc() + resultCount.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, strconv.FormatBool(result.Success)).Inc() }