From ae2c4b1ea98776d91ee33344a2dfa7602b034484 Mon Sep 17 00:00:00 2001 From: TwinProduction Date: Sat, 19 Sep 2020 16:22:12 -0400 Subject: [PATCH] Start working on #12: Fix inconsistencies in alerting providers --- README.md | 14 +- alerting/config.go | 8 + alerting/custom.go | 80 ++++++++ .../custom_test.go | 13 +- alerting/pagerduty.go | 36 +++- alerting/pagerduty_test.go | 14 ++ alerting/slack.go | 59 ++++++ alerting/slack_test.go | 14 ++ alerting/twilio.go | 34 ++++ alerting/twilio_test.go | 19 ++ config/config.go | 9 +- config/config_test.go | 5 +- core/alerting.go | 178 ------------------ docs/pagerduty-integration-guide.md | 5 +- {alerting => watchdog}/alerting.go | 86 ++++----- watchdog/watchdog.go | 3 +- 16 files changed, 337 insertions(+), 240 deletions(-) create mode 100644 alerting/config.go create mode 100644 alerting/custom.go rename core/alerting_test.go => alerting/custom_test.go (81%) create mode 100644 alerting/pagerduty_test.go create mode 100644 alerting/slack.go create mode 100644 alerting/slack_test.go create mode 100644 alerting/twilio.go create mode 100644 alerting/twilio_test.go delete mode 100644 core/alerting.go rename {alerting => watchdog}/alerting.go (61%) diff --git a/README.md b/README.md index 705a1399..4d140e73 100644 --- a/README.md +++ b/README.md @@ -94,15 +94,17 @@ Note that you can also add environment variables in the configuration file (i.e. | `services[].alerts[].send-on-resolved` | Whether to send a notification once a triggered alert is marked as resolved | `false` | | `services[].alerts[].description` | Description of the alert. Will be included in the alert sent | `""` | | `alerting` | Configuration for alerting | `{}` | -| `alerting.slack` | Webhook to use for alerts of type `slack` | `""` | -| `alerting.pagerduty` | PagerDuty Events API v2 integration key. Used for alerts of type `pagerduty` | `""` | +| `alerting.slack` | Configuration for alerts of type `slack` | `""` | +| `alerting.slack.webhook-url` | Slack Webhook URL | Required `""` | +| `alerting.pagerduty` | Configuration for alerts of type `pagerduty` | `""` | +| `alerting.pagerduty.integration-key` | PagerDuty Events API v2 integration key. | Required `""` | | `alerting.twilio` | Settings for alerts of type `twilio` | `""` | | `alerting.twilio.sid` | Twilio account SID | Required `""` | | `alerting.twilio.token` | Twilio auth token | Required `""` | | `alerting.twilio.from` | Number to send Twilio alerts from | Required `""` | | `alerting.twilio.to` | Number to send twilio alerts to | Required `""` | | `alerting.custom` | Configuration for custom actions on failure or alerts | `""` | -| `alerting.custom.url` | Custom alerting request url | `""` | +| `alerting.custom.url` | Custom alerting request url | Required `""` | | `alerting.custom.body` | Custom alerting request body. | `""` | | `alerting.custom.headers` | Custom alerting request headers | `{}` | @@ -133,7 +135,8 @@ Here are some examples of conditions you can use: ```yaml alerting: - slack: "https://hooks.slack.com/services/**********/**********/**********" + slack: + webhook-url: "https://hooks.slack.com/services/**********/**********/**********" services: - name: twinnation interval: 30s @@ -168,7 +171,8 @@ PagerDuty instead. ```yaml alerting: - pagerduty: "********************************" + pagerduty: + integration-key: "********************************" services: - name: twinnation interval: 30s diff --git a/alerting/config.go b/alerting/config.go new file mode 100644 index 00000000..e19a465f --- /dev/null +++ b/alerting/config.go @@ -0,0 +1,8 @@ +package alerting + +type Config struct { + Slack *SlackAlertProvider `yaml:"slack"` + PagerDuty *PagerDutyAlertProvider `yaml:"pagerduty"` + Twilio *TwilioAlertProvider `yaml:"twilio"` + Custom *CustomAlertProvider `yaml:"custom"` +} diff --git a/alerting/custom.go b/alerting/custom.go new file mode 100644 index 00000000..313f0e28 --- /dev/null +++ b/alerting/custom.go @@ -0,0 +1,80 @@ +package alerting + +import ( + "bytes" + "fmt" + "github.com/TwinProduction/gatus/client" + "io/ioutil" + "net/http" + "strings" +) + +type CustomAlertProvider struct { + Url string `yaml:"url"` + Method string `yaml:"method,omitempty"` + Body string `yaml:"body,omitempty"` + Headers map[string]string `yaml:"headers,omitempty"` +} + +func (provider *CustomAlertProvider) IsValid() bool { + return len(provider.Url) > 0 +} + +func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request { + body := provider.Body + providerUrl := provider.Url + method := provider.Method + if strings.Contains(body, "[ALERT_DESCRIPTION]") { + body = strings.ReplaceAll(body, "[ALERT_DESCRIPTION]", alertDescription) + } + if strings.Contains(body, "[SERVICE_NAME]") { + body = strings.ReplaceAll(body, "[SERVICE_NAME]", serviceName) + } + if strings.Contains(body, "[ALERT_TRIGGERED_OR_RESOLVED]") { + if resolved { + body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED") + } else { + body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED") + } + } + if strings.Contains(providerUrl, "[ALERT_DESCRIPTION]") { + providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_DESCRIPTION]", alertDescription) + } + if strings.Contains(providerUrl, "[SERVICE_NAME]") { + providerUrl = strings.ReplaceAll(providerUrl, "[SERVICE_NAME]", serviceName) + } + if strings.Contains(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]") { + if resolved { + providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED") + } else { + providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED") + } + } + if len(method) == 0 { + method = "GET" + } + bodyBuffer := bytes.NewBuffer([]byte(body)) + request, _ := http.NewRequest(method, providerUrl, bodyBuffer) + for k, v := range provider.Headers { + request.Header.Set(k, v) + } + return request +} + +// Send a request to the alert provider and return the body +func (provider *CustomAlertProvider) Send(serviceName, alertDescription string, resolved bool) ([]byte, error) { + request := provider.buildRequest(serviceName, alertDescription, resolved) + response, err := client.GetHttpClient().Do(request) + if err != nil { + return nil, err + } + if response.StatusCode > 399 { + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, fmt.Errorf("call to provider alert returned status code %d", response.StatusCode) + } else { + return nil, fmt.Errorf("call to provider alert returned status code %d: %s", response.StatusCode, string(body)) + } + } + return ioutil.ReadAll(response.Body) +} diff --git a/core/alerting_test.go b/alerting/custom_test.go similarity index 81% rename from core/alerting_test.go rename to alerting/custom_test.go index ff0ecad9..20ca53ee 100644 --- a/core/alerting_test.go +++ b/alerting/custom_test.go @@ -1,10 +1,21 @@ -package core +package alerting import ( "io/ioutil" "testing" ) +func TestCustomAlertProvider_IsValid(t *testing.T) { + invalidProvider := CustomAlertProvider{Url: ""} + if invalidProvider.IsValid() { + t.Error("provider shouldn't have been valid") + } + validProvider := CustomAlertProvider{Url: "http://example.com"} + if !validProvider.IsValid() { + t.Error("provider should've been valid") + } +} + func TestCustomAlertProvider_buildRequestWhenResolved(t *testing.T) { const ( ExpectedUrl = "http://example.com/service-name" diff --git a/alerting/pagerduty.go b/alerting/pagerduty.go index 9ef4df03..2657e979 100644 --- a/alerting/pagerduty.go +++ b/alerting/pagerduty.go @@ -1,7 +1,35 @@ package alerting -type pagerDutyResponse struct { - Status string `json:"status"` - Message string `json:"message"` - DedupKey string `json:"dedup_key"` +import ( + "fmt" + "github.com/TwinProduction/gatus/core" +) + +type PagerDutyAlertProvider struct { + IntegrationKey string `yaml:"integration-key"` +} + +func (provider *PagerDutyAlertProvider) IsValid() bool { + return len(provider.IntegrationKey) == 32 +} + +// https://developer.pagerduty.com/docs/events-api-v2/trigger-events/ +func (provider *PagerDutyAlertProvider) ToCustomAlertProvider(eventAction, resolveKey string, service *core.Service, message string) *CustomAlertProvider { + return &CustomAlertProvider{ + Url: "https://events.pagerduty.com/v2/enqueue", + Method: "POST", + Body: fmt.Sprintf(`{ + "routing_key": "%s", + "dedup_key": "%s", + "event_action": "%s", + "payload": { + "summary": "%s", + "source": "%s", + "severity": "critical" + } +}`, provider.IntegrationKey, resolveKey, eventAction, message, service.Name), + Headers: map[string]string{ + "Content-Type": "application/json", + }, + } } diff --git a/alerting/pagerduty_test.go b/alerting/pagerduty_test.go new file mode 100644 index 00000000..338926cd --- /dev/null +++ b/alerting/pagerduty_test.go @@ -0,0 +1,14 @@ +package alerting + +import "testing" + +func TestPagerDutyAlertProvider_IsValid(t *testing.T) { + invalidProvider := PagerDutyAlertProvider{IntegrationKey: ""} + if invalidProvider.IsValid() { + t.Error("provider shouldn't have been valid") + } + validProvider := PagerDutyAlertProvider{IntegrationKey: "00000000000000000000000000000000"} + if !validProvider.IsValid() { + t.Error("provider should've been valid") + } +} diff --git a/alerting/slack.go b/alerting/slack.go new file mode 100644 index 00000000..4abdc361 --- /dev/null +++ b/alerting/slack.go @@ -0,0 +1,59 @@ +package alerting + +import ( + "fmt" + "github.com/TwinProduction/gatus/core" +) + +type SlackAlertProvider struct { + WebhookUrl string `yaml:"webhook-url"` +} + +func (provider *SlackAlertProvider) IsValid() bool { + return len(provider.WebhookUrl) > 0 +} + +func (provider *SlackAlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *CustomAlertProvider { + var message string + var color string + if resolved { + message = fmt.Sprintf("An alert for *%s* has been resolved after passing successfully %d time(s) in a row", service.Name, alert.SuccessThreshold) + color = "#36A64F" + } else { + message = fmt.Sprintf("An alert for *%s* has been triggered due to having failed %d time(s) in a row", service.Name, alert.FailureThreshold) + color = "#DD0000" + } + var results string + for _, conditionResult := range result.ConditionResults { + var prefix string + if conditionResult.Success { + prefix = ":heavy_check_mark:" + } else { + prefix = ":x:" + } + results += fmt.Sprintf("%s - `%s`\n", prefix, conditionResult.Condition) + } + return &CustomAlertProvider{ + Url: provider.WebhookUrl, + Method: "POST", + Body: fmt.Sprintf(`{ + "text": "", + "attachments": [ + { + "title": ":helmet_with_white_cross: Gatus", + "text": "%s:\n> %s", + "short": false, + "color": "%s", + "fields": [ + { + "title": "Condition results", + "value": "%s", + "short": false + } + ] + }, + ] +}`, message, alert.Description, color, results), + Headers: map[string]string{"Content-Type": "application/json"}, + } +} diff --git a/alerting/slack_test.go b/alerting/slack_test.go new file mode 100644 index 00000000..8d211caa --- /dev/null +++ b/alerting/slack_test.go @@ -0,0 +1,14 @@ +package alerting + +import "testing" + +func TestSlackAlertProvider_IsValid(t *testing.T) { + invalidProvider := SlackAlertProvider{WebhookUrl: ""} + if invalidProvider.IsValid() { + t.Error("provider shouldn't have been valid") + } + validProvider := SlackAlertProvider{WebhookUrl: "http://example.com"} + if !validProvider.IsValid() { + t.Error("provider should've been valid") + } +} diff --git a/alerting/twilio.go b/alerting/twilio.go new file mode 100644 index 00000000..933122fc --- /dev/null +++ b/alerting/twilio.go @@ -0,0 +1,34 @@ +package alerting + +import ( + "encoding/base64" + "fmt" + "net/url" +) + +type TwilioAlertProvider struct { + SID string `yaml:"sid"` + Token string `yaml:"token"` + From string `yaml:"from"` + To string `yaml:"to"` +} + +func (provider *TwilioAlertProvider) IsValid() bool { + return len(provider.Token) > 0 && len(provider.SID) > 0 && len(provider.From) > 0 && len(provider.To) > 0 +} + +func (provider *TwilioAlertProvider) ToCustomAlertProvider(message string) *CustomAlertProvider { + return &CustomAlertProvider{ + Url: fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID), + Method: "POST", + Body: url.Values{ + "To": {provider.To}, + "From": {provider.From}, + "Body": {message}, + }.Encode(), + Headers: map[string]string{ + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", provider.SID, provider.Token)))), + }, + } +} diff --git a/alerting/twilio_test.go b/alerting/twilio_test.go new file mode 100644 index 00000000..1da7d4aa --- /dev/null +++ b/alerting/twilio_test.go @@ -0,0 +1,19 @@ +package alerting + +import "testing" + +func TestTwilioAlertProvider_IsValid(t *testing.T) { + invalidProvider := TwilioAlertProvider{} + if invalidProvider.IsValid() { + t.Error("provider shouldn't have been valid") + } + validProvider := TwilioAlertProvider{ + SID: "1", + Token: "1", + From: "1", + To: "1", + } + if !validProvider.IsValid() { + t.Error("provider should've been valid") + } +} diff --git a/config/config.go b/config/config.go index 2ba7fe9d..a23d97e4 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( "errors" + "github.com/TwinProduction/gatus/alerting" "github.com/TwinProduction/gatus/core" "gopkg.in/yaml.v2" "io/ioutil" @@ -21,10 +22,10 @@ var ( ) type Config struct { - Metrics bool `yaml:"metrics"` - Debug bool `yaml:"debug"` - Alerting *core.AlertingConfig `yaml:"alerting"` - Services []*core.Service `yaml:"services"` + Metrics bool `yaml:"metrics"` + Debug bool `yaml:"debug"` + Alerting *alerting.Config `yaml:"alerting"` + Services []*core.Service `yaml:"services"` } func Get() *Config { diff --git a/config/config_test.go b/config/config_test.go index 4a87c176..847d16f5 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -121,7 +121,8 @@ badconfig: func TestParseAndValidateConfigBytesWithAlerting(t *testing.T) { config, err := parseAndValidateConfigBytes([]byte(` alerting: - slack: "http://example.com" + slack: + webhook-url: "http://example.com" services: - name: twinnation url: https://twinnation.org/actuator/health @@ -145,7 +146,7 @@ services: if config.Alerting == nil { t.Fatal("config.AlertingConfig shouldn't have been nil") } - if config.Alerting.Slack != "http://example.com" { + if config.Alerting.Slack.WebhookUrl != "http://example.com" { t.Errorf("Slack webhook should've been %s, but was %s", "http://example.com", config.Alerting.Slack) } if len(config.Services) != 1 { diff --git a/core/alerting.go b/core/alerting.go deleted file mode 100644 index f2328fc2..00000000 --- a/core/alerting.go +++ /dev/null @@ -1,178 +0,0 @@ -package core - -import ( - "bytes" - "encoding/base64" - "fmt" - "github.com/TwinProduction/gatus/client" - "io/ioutil" - "net/http" - "net/url" - "strings" -) - -type AlertingConfig struct { - Slack string `yaml:"slack"` - PagerDuty string `yaml:"pagerduty"` - Twilio *TwilioAlertProvider `yaml:"twilio"` - Custom *CustomAlertProvider `yaml:"custom"` -} - -type TwilioAlertProvider struct { - SID string `yaml:"sid"` - Token string `yaml:"token"` - From string `yaml:"from"` - To string `yaml:"to"` -} - -func (provider *TwilioAlertProvider) IsValid() bool { - return len(provider.Token) > 0 && len(provider.SID) > 0 && len(provider.From) > 0 && len(provider.To) > 0 -} - -type CustomAlertProvider struct { - Url string `yaml:"url"` - Method string `yaml:"method,omitempty"` - Body string `yaml:"body,omitempty"` - Headers map[string]string `yaml:"headers,omitempty"` -} - -func (provider *CustomAlertProvider) IsValid() bool { - return len(provider.Url) > 0 -} - -func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request { - body := provider.Body - providerUrl := provider.Url - if strings.Contains(body, "[ALERT_DESCRIPTION]") { - body = strings.ReplaceAll(body, "[ALERT_DESCRIPTION]", alertDescription) - } - if strings.Contains(body, "[SERVICE_NAME]") { - body = strings.ReplaceAll(body, "[SERVICE_NAME]", serviceName) - } - if strings.Contains(body, "[ALERT_TRIGGERED_OR_RESOLVED]") { - if resolved { - body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED") - } else { - body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED") - } - } - if strings.Contains(providerUrl, "[ALERT_DESCRIPTION]") { - providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_DESCRIPTION]", alertDescription) - } - if strings.Contains(providerUrl, "[SERVICE_NAME]") { - providerUrl = strings.ReplaceAll(providerUrl, "[SERVICE_NAME]", serviceName) - } - if strings.Contains(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]") { - if resolved { - providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED") - } else { - providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED") - } - } - bodyBuffer := bytes.NewBuffer([]byte(body)) - request, _ := http.NewRequest(provider.Method, providerUrl, bodyBuffer) - for k, v := range provider.Headers { - request.Header.Set(k, v) - } - return request -} - -// Send a request to the alert provider and return the body -func (provider *CustomAlertProvider) Send(serviceName, alertDescription string, resolved bool) ([]byte, error) { - request := provider.buildRequest(serviceName, alertDescription, resolved) - response, err := client.GetHttpClient().Do(request) - if err != nil { - return nil, err - } - if response.StatusCode > 399 { - body, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, fmt.Errorf("call to provider alert returned status code %d", response.StatusCode) - } else { - return nil, fmt.Errorf("call to provider alert returned status code %d: %s", response.StatusCode, string(body)) - } - } - return ioutil.ReadAll(response.Body) -} - -func CreateSlackCustomAlertProvider(slackWebHookUrl string, service *Service, alert *Alert, result *Result, resolved bool) *CustomAlertProvider { - var message string - var color string - if resolved { - message = fmt.Sprintf("An alert for *%s* has been resolved after passing successfully %d time(s) in a row", service.Name, alert.SuccessThreshold) - color = "#36A64F" - } else { - message = fmt.Sprintf("An alert for *%s* has been triggered due to having failed %d time(s) in a row", service.Name, alert.FailureThreshold) - color = "#DD0000" - } - var results string - for _, conditionResult := range result.ConditionResults { - var prefix string - if conditionResult.Success { - prefix = ":heavy_check_mark:" - } else { - prefix = ":x:" - } - results += fmt.Sprintf("%s - `%s`\n", prefix, conditionResult.Condition) - } - return &CustomAlertProvider{ - Url: slackWebHookUrl, - Method: "POST", - Body: fmt.Sprintf(`{ - "text": "", - "attachments": [ - { - "title": ":helmet_with_white_cross: Gatus", - "text": "%s:\n> %s", - "short": false, - "color": "%s", - "fields": [ - { - "title": "Condition results", - "value": "%s", - "short": false - } - ] - }, - ] -}`, message, alert.Description, color, results), - Headers: map[string]string{"Content-Type": "application/json"}, - } -} - -func CreateTwilioCustomAlertProvider(provider *TwilioAlertProvider, message string) *CustomAlertProvider { - return &CustomAlertProvider{ - Url: fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID), - Method: "POST", - Body: url.Values{ - "To": {provider.To}, - "From": {provider.From}, - "Body": {message}, - }.Encode(), - Headers: map[string]string{ - "Content-Type": "application/x-www-form-urlencoded", - "Authorization": fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", provider.SID, provider.Token)))), - }, - } -} - -// https://developer.pagerduty.com/docs/events-api-v2/trigger-events/ -func CreatePagerDutyCustomAlertProvider(routingKey, eventAction, resolveKey string, service *Service, message string) *CustomAlertProvider { - return &CustomAlertProvider{ - Url: "https://events.pagerduty.com/v2/enqueue", - Method: "POST", - Body: fmt.Sprintf(`{ - "routing_key": "%s", - "dedup_key": "%s", - "event_action": "%s", - "payload": { - "summary": "%s", - "source": "%s", - "severity": "critical" - } -}`, routingKey, resolveKey, eventAction, message, service.Name), - Headers: map[string]string{ - "Content-Type": "application/json", - }, - } -} diff --git a/docs/pagerduty-integration-guide.md b/docs/pagerduty-integration-guide.md index 2e1b29b4..8b912ac8 100644 --- a/docs/pagerduty-integration-guide.md +++ b/docs/pagerduty-integration-guide.md @@ -30,10 +30,11 @@ If you need help with this integration, please create an issue at https://github ## In Gatus -In your configuration file, you must first specify the integration key in `alerting.pagerduty`, like so: +In your configuration file, you must first specify the integration key at `alerting.pagerduty.integration-key`, like so: ```yaml alerting: - pagerduty: "********************************" + pagerduty: + integration-key: "********************************" ``` You can now add alerts of type `pagerduty` in the services you've defined, like so: ```yaml diff --git a/alerting/alerting.go b/watchdog/alerting.go similarity index 61% rename from alerting/alerting.go rename to watchdog/alerting.go index 59605456..634408dc 100644 --- a/alerting/alerting.go +++ b/watchdog/alerting.go @@ -1,15 +1,16 @@ -package alerting +package watchdog import ( "encoding/json" "fmt" + "github.com/TwinProduction/gatus/alerting" "github.com/TwinProduction/gatus/config" "github.com/TwinProduction/gatus/core" "log" ) -// Handle takes care of alerts to resolve and alerts to trigger based on result success or failure -func Handle(service *core.Service, result *core.Result) { +// HandleAlerting takes care of alerts to resolve and alerts to trigger based on result success or failure +func HandleAlerting(service *core.Service, result *core.Result) { cfg := config.Get() if cfg.Alerting == nil { return @@ -31,43 +32,38 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf } if alert.Triggered { if cfg.Debug { - log.Printf("[alerting][handleAlertsToTrigger] Alert with description='%s' has already been triggered, skipping", alert.Description) + log.Printf("[watchdog][handleAlertsToTrigger] Alert with description='%s' has already been triggered, skipping", alert.Description) } continue } - var alertProvider *core.CustomAlertProvider + var alertProvider *alerting.CustomAlertProvider if alert.Type == core.SlackAlert { - if len(cfg.Alerting.Slack) > 0 { - log.Printf("[alerting][handleAlertsToTrigger] Sending Slack alert because alert with description='%s' has been triggered", alert.Description) - alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, false) + if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() { + log.Printf("[watchdog][handleAlertsToTrigger] Sending Slack alert because alert with description='%s' has been triggered", alert.Description) + alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, false) } else { - log.Printf("[alerting][handleAlertsToTrigger] Not sending Slack alert despite being triggered, because there is no Slack webhook configured") + log.Printf("[watchdog][handleAlertsToTrigger] Not sending Slack alert despite being triggered, because there is no Slack webhook configured") } } else if alert.Type == core.PagerDutyAlert { - if len(cfg.Alerting.PagerDuty) > 0 { - log.Printf("[alerting][handleAlertsToTrigger] Sending PagerDuty alert because alert with description='%s' has been triggered", alert.Description) - alertProvider = core.CreatePagerDutyCustomAlertProvider(cfg.Alerting.PagerDuty, "trigger", "", service, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description)) + if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() { + log.Printf("[watchdog][handleAlertsToTrigger] Sending PagerDuty alert because alert with description='%s' has been triggered", alert.Description) + alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("trigger", "", service, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description)) } else { - log.Printf("[alerting][handleAlertsToTrigger] Not sending PagerDuty alert despite being triggered, because PagerDuty isn't configured properly") + log.Printf("[watchdog][handleAlertsToTrigger] Not sending PagerDuty alert despite being triggered, because PagerDuty isn't configured properly") } } else if alert.Type == core.TwilioAlert { if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() { - log.Printf("[alerting][handleAlertsToTrigger] Sending Twilio alert because alert with description='%s' has been triggered", alert.Description) - alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description)) + log.Printf("[watchdog][handleAlertsToTrigger] Sending Twilio alert because alert with description='%s' has been triggered", alert.Description) + alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description)) } else { - log.Printf("[alerting][handleAlertsToTrigger] Not sending Twilio alert despite being triggered, because Twilio config settings missing") + log.Printf("[watchdog][handleAlertsToTrigger] Not sending Twilio alert despite being triggered, because Twilio config settings missing") } } else if alert.Type == core.CustomAlert { if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() { - log.Printf("[alerting][handleAlertsToTrigger] Sending custom alert because alert with description='%s' has been triggered", alert.Description) - alertProvider = &core.CustomAlertProvider{ - Url: cfg.Alerting.Custom.Url, - Method: cfg.Alerting.Custom.Method, - Body: cfg.Alerting.Custom.Body, - Headers: cfg.Alerting.Custom.Headers, - } + log.Printf("[watchdog][handleAlertsToTrigger] Sending custom alert because alert with description='%s' has been triggered", alert.Description) + alertProvider = cfg.Alerting.Custom } else { - log.Printf("[alerting][handleAlertsToTrigger] Not sending custom alert despite being triggered, because there is no custom url configured") + log.Printf("[watchdog][handleAlertsToTrigger] Not sending custom alert despite being triggered, because there is no custom url configured") } } if alertProvider != nil { @@ -80,7 +76,7 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf var response pagerDutyResponse err = json.Unmarshal(body, &response) if err != nil { - log.Printf("[alerting][handleAlertsToTrigger] Ran into error unmarshaling pager duty response: %s", err.Error()) + log.Printf("[watchdog][handleAlertsToTrigger] Ran into error unmarshaling pager duty response: %s", err.Error()) } else { alert.ResolveKey = response.DedupKey } @@ -89,7 +85,7 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf _, err = alertProvider.Send(service.Name, alert.Description, false) } if err != nil { - log.Printf("[alerting][handleAlertsToTrigger] Ran into error sending an alert: %s", err.Error()) + log.Printf("[watchdog][handleAlertsToTrigger] Ran into error sending an alert: %s", err.Error()) } else { alert.Triggered = true } @@ -107,46 +103,46 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf if !alert.SendOnResolved { continue } - var alertProvider *core.CustomAlertProvider + var alertProvider *alerting.CustomAlertProvider if alert.Type == core.SlackAlert { - if len(cfg.Alerting.Slack) > 0 { - log.Printf("[alerting][handleAlertsToResolve] Sending Slack alert because alert with description='%s' has been resolved", alert.Description) - alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, true) + if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() { + log.Printf("[watchdog][handleAlertsToResolve] Sending Slack alert because alert with description='%s' has been resolved", alert.Description) + alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, true) } else { - log.Printf("[alerting][handleAlertsToResolve] Not sending Slack alert despite being resolved, because there is no Slack webhook configured") + log.Printf("[watchdog][handleAlertsToResolve] Not sending Slack alert despite being resolved, because there is no Slack webhook configured") } } else if alert.Type == core.PagerDutyAlert { - if len(cfg.Alerting.PagerDuty) > 0 { - log.Printf("[alerting][handleAlertsToResolve] Sending PagerDuty alert because alert with description='%s' has been resolved", alert.Description) - alertProvider = core.CreatePagerDutyCustomAlertProvider(cfg.Alerting.PagerDuty, "resolve", alert.ResolveKey, service, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description)) + if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() { + log.Printf("[watchdog][handleAlertsToResolve] Sending PagerDuty alert because alert with description='%s' has been resolved", alert.Description) + alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("resolve", alert.ResolveKey, service, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description)) } else { - log.Printf("[alerting][handleAlertsToResolve] Not sending PagerDuty alert despite being resolved, because PagerDuty isn't configured properly") + log.Printf("[watchdog][handleAlertsToResolve] Not sending PagerDuty alert despite being resolved, because PagerDuty isn't configured properly") } } else if alert.Type == core.TwilioAlert { if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() { - log.Printf("[alerting][handleAlertsToResolve] Sending Twilio alert because alert with description='%s' has been resolved", alert.Description) - alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description)) + log.Printf("[watchdog][handleAlertsToResolve] Sending Twilio alert because alert with description='%s' has been resolved", alert.Description) + alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description)) } else { - log.Printf("[alerting][handleAlertsToResolve] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly") + log.Printf("[watchdog][handleAlertsToResolve] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly") } } else if alert.Type == core.CustomAlert { if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() { - log.Printf("[alerting][handleAlertsToResolve] Sending custom alert because alert with description='%s' has been resolved", alert.Description) - alertProvider = &core.CustomAlertProvider{ + log.Printf("[watchdog][handleAlertsToResolve] Sending custom alert because alert with description='%s' has been resolved", alert.Description) + alertProvider = &alerting.CustomAlertProvider{ Url: cfg.Alerting.Custom.Url, Method: cfg.Alerting.Custom.Method, Body: cfg.Alerting.Custom.Body, Headers: cfg.Alerting.Custom.Headers, } } else { - log.Printf("[alerting][handleAlertsToResolve] Not sending custom alert despite being resolved, because the custom provider isn't configured properly") + log.Printf("[watchdog][handleAlertsToResolve] Not sending custom alert despite being resolved, because the custom provider isn't configured properly") } } if alertProvider != nil { // TODO: retry on error _, err := alertProvider.Send(service.Name, alert.Description, true) if err != nil { - log.Printf("[alerting][handleAlertsToResolve] Ran into error sending an alert: %s", err.Error()) + log.Printf("[watchdog][handleAlertsToResolve] Ran into error sending an alert: %s", err.Error()) } else { if alert.Type == core.PagerDutyAlert { alert.ResolveKey = "" @@ -156,3 +152,9 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf } service.NumberOfFailuresInARow = 0 } + +type pagerDutyResponse struct { + Status string `json:"status"` + Message string `json:"message"` + DedupKey string `json:"dedup_key"` +} diff --git a/watchdog/watchdog.go b/watchdog/watchdog.go index 8d4f6a97..246e7bd6 100644 --- a/watchdog/watchdog.go +++ b/watchdog/watchdog.go @@ -3,7 +3,6 @@ package watchdog import ( "encoding/json" "fmt" - "github.com/TwinProduction/gatus/alerting" "github.com/TwinProduction/gatus/config" "github.com/TwinProduction/gatus/core" "github.com/TwinProduction/gatus/metric" @@ -71,7 +70,7 @@ func monitor(service *core.Service) { result.Duration.Round(time.Millisecond), extra, ) - alerting.Handle(service, result) + HandleAlerting(service, result) if cfg.Debug { log.Printf("[watchdog][monitor] Waiting for interval=%s before monitoring serviceName=%s again", service.Interval, service.Name) }