BREAKING: Replace metrics

This commit is contained in:
TwiN 2021-12-13 19:24:18 -05:00
parent eb9acef9b5
commit b67701ff6d
3 changed files with 228 additions and 113 deletions

View File

@ -0,0 +1,23 @@
## Usage
```console
docker-compose up
```
Once you've done the above, you should be able to access the Grafana dashboard at `http://localhost:3000`.
## Queries
Gatus uses Prometheus counters.
Total results per minute:
```
sum(rate(gatus_results_total[5m])*60) by (key)
```
Total successful results per minute:
```
sum(rate(gatus_results_total{success="true"}[5m])*60) by (key)
```
Total unsuccessful results per minute:
```
sum(rate(gatus_results_total{success="true"}[5m])*60) by (key)
```

View File

@ -15,7 +15,6 @@
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 2,
"links": [],
"panels": [
{
@ -24,20 +23,26 @@
"dashLength": 10,
"dashes": false,
"datasource": null,
"description": "Number of results per minute",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 14,
"h": 7,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"interval": "",
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"hideEmpty": false,
"hideZero": false,
"max": false,
"min": false,
"rightSide": false,
"show": true,
"total": false,
"values": false
@ -58,9 +63,13 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(gatus_tasks[30s])) by (endpoint)",
"expr": "sum(rate(gatus_results_total[5m])*60) by (key)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "30s",
"legendFormat": "{{endpoint}}",
"intervalFactor": 1,
"legendFormat": "{{key}}",
"refId": "A"
}
],
@ -68,7 +77,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "All tasks executed over time",
"title": "Total results per minute",
"tooltip": {
"shared": true,
"sort": 0,
@ -84,6 +93,7 @@
},
"yaxes": [
{
"decimals": null,
"format": "short",
"label": null,
"logBase": 1,
@ -119,93 +129,6 @@
"x": 12,
"y": 0
},
"id": 3,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(gatus_tasks{success=\"false\"}[30s])) by (endpoint)",
"interval": "30s",
"legendFormat": "{{endpoint}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Unsuccessful tasks",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 7
},
"id": 5,
"legend": {
"avg": false,
@ -232,10 +155,10 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(gatus_tasks{success=\"true\"}[30s])) by (endpoint)",
"expr": "sum(rate(gatus_results_total{success=\"true\"}[5m])*60) by (key)",
"instant": false,
"interval": "30s",
"legendFormat": "{{endpoint}}",
"legendFormat": "{{key}}",
"refId": "A"
}
],
@ -243,7 +166,183 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Successful tasks",
"title": "Successful results per minute",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"cacheTimeout": null,
"colorBackground": false,
"colorPostfix": false,
"colorPrefix": false,
"colorValue": true,
"colors": [
"#299c46",
"rgba(237, 129, 40, 0.89)",
"#d44a3a"
],
"datasource": null,
"format": "none",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 7
},
"id": 7,
"interval": "",
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"options": {},
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": true,
"ymax": null,
"ymin": null
},
"tableColumn": "",
"targets": [
{
"expr": "rate(gatus_results_total{success=\"false\"}[1m])*60",
"format": "time_series",
"instant": false,
"intervalFactor": 1,
"refId": "A"
}
],
"thresholds": "1,2",
"timeFrom": null,
"timeShift": null,
"title": "Unsuccessful results",
"type": "singlestat",
"valueFontSize": "150%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "current"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 7
},
"id": 3,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(gatus_results_total{success=\"false\"}[5m])*60) by (key)",
"interval": "30s",
"legendFormat": "{{key}} ",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Unsuccessful results per minute",
"tooltip": {
"shared": true,
"sort": 0,
@ -289,12 +388,12 @@
"list": []
},
"time": {
"from": "now-30m",
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Gatus",
"uid": "KPI7Qj1Wk",
"version": 3
"version": 1
}

View File

@ -1,9 +1,7 @@
package metric
import (
"fmt"
"strconv"
"sync"
"github.com/TwiN/gatus/v3/core"
"github.com/prometheus/client_golang/prometheus"
@ -11,24 +9,19 @@ import (
)
var (
gauges = map[string]*prometheus.GaugeVec{}
rwLock sync.RWMutex
// This will be initialized once PublishMetricsForEndpoint.
// The reason why we're doing this is that if metrics are disabled, we don't want to initialize it unnecessarily.
resultCount *prometheus.CounterVec = nil
)
// PublishMetricsForEndpoint publishes metrics for the given endpoint and its result.
// These metrics will be exposed at /metrics if the metrics are enabled
func PublishMetricsForEndpoint(endpoint *core.Endpoint, result *core.Result) {
rwLock.Lock()
gauge, exists := gauges[fmt.Sprintf("%s_%s", endpoint.Name, endpoint.URL)]
if !exists {
gauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
Subsystem: "gatus",
Name: "tasks",
// TODO: remove the "service" key in v4.0.0, as it is only kept for backward compatibility
ConstLabels: prometheus.Labels{"service": endpoint.Name, "endpoint": endpoint.Name, "url": endpoint.URL},
}, []string{"status", "success"})
gauges[fmt.Sprintf("%s_%s", endpoint.Name, endpoint.URL)] = gauge
if resultCount == nil {
resultCount = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "gatus_results_total",
Help: "Number of results per endpoint",
}, []string{"key", "group", "name", "success"})
}
rwLock.Unlock()
gauge.WithLabelValues(strconv.Itoa(result.HTTPStatus), strconv.FormatBool(result.Success)).Inc()
resultCount.WithLabelValues(endpoint.Key(), endpoint.Group, endpoint.Name, strconv.FormatBool(result.Success)).Inc()
}