diff --git a/alerting/provider/custom/custom.go b/alerting/provider/custom/custom.go index 0885a01b..6563f252 100644 --- a/alerting/provider/custom/custom.go +++ b/alerting/provider/custom/custom.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "github.com/TwinProduction/gatus/client" + "github.com/TwinProduction/gatus/core" "io/ioutil" "net/http" "strings" @@ -20,6 +21,10 @@ func (provider *AlertProvider) IsValid() bool { return len(provider.Url) > 0 } +func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *AlertProvider { + return provider +} + func (provider *AlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request { body := provider.Body providerUrl := provider.Url diff --git a/alerting/provider/pagerduty/pagerduty.go b/alerting/provider/pagerduty/pagerduty.go index d104dfbc..fbda22f2 100644 --- a/alerting/provider/pagerduty/pagerduty.go +++ b/alerting/provider/pagerduty/pagerduty.go @@ -15,7 +15,17 @@ func (provider *AlertProvider) IsValid() bool { } // https://developer.pagerduty.com/docs/events-api-v2/trigger-events/ -func (provider *AlertProvider) ToCustomAlertProvider(eventAction, resolveKey string, service *core.Service, message string) *custom.AlertProvider { +func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *custom.AlertProvider { + var message, eventAction, resolveKey string + if resolved { + message = fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description) + eventAction = "resolve" + resolveKey = alert.ResolveKey + } else { + message = fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description) + eventAction = "trigger" + resolveKey = "" + } return &custom.AlertProvider{ Url: "https://events.pagerduty.com/v2/enqueue", Method: "POST", diff --git a/alerting/provider/provider.go b/alerting/provider/provider.go index 10096d89..f9783a7b 100644 --- a/alerting/provider/provider.go +++ b/alerting/provider/provider.go @@ -1,5 +1,11 @@ package provider +import ( + "github.com/TwinProduction/gatus/alerting/provider/custom" + "github.com/TwinProduction/gatus/core" +) + type AlertProvider interface { IsValid() bool + ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *custom.AlertProvider } diff --git a/alerting/provider/twilio/twilio.go b/alerting/provider/twilio/twilio.go index 7d5c13ba..6e5e06aa 100644 --- a/alerting/provider/twilio/twilio.go +++ b/alerting/provider/twilio/twilio.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "fmt" "github.com/TwinProduction/gatus/alerting/provider/custom" + "github.com/TwinProduction/gatus/core" "net/url" ) @@ -18,7 +19,13 @@ func (provider *AlertProvider) IsValid() bool { return len(provider.Token) > 0 && len(provider.SID) > 0 && len(provider.From) > 0 && len(provider.To) > 0 } -func (provider *AlertProvider) ToCustomAlertProvider(message string) *custom.AlertProvider { +func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *custom.AlertProvider { + var message string + if resolved { + message = fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description) + } else { + message = fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description) + } return &custom.AlertProvider{ Url: fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID), Method: "POST", diff --git a/client/client_test.go b/client/client_test.go new file mode 100644 index 00000000..0ab546e4 --- /dev/null +++ b/client/client_test.go @@ -0,0 +1,13 @@ +package client + +import "testing" + +func TestGetHttpClient(t *testing.T) { + if client != nil { + t.Error("client should've been nil since it hasn't been called a single time yet") + } + _ = GetHttpClient() + if client == nil { + t.Error("client shouldn't have been nil, since it has been called once") + } +} diff --git a/config.yaml b/config.yaml index 41ec50cd..cca4c053 100644 --- a/config.yaml +++ b/config.yaml @@ -1,15 +1,15 @@ metrics: true services: - name: twinnation + url: "https://twinnation.org/health" interval: 30s - url: https://twinnation.org/health conditions: - "[STATUS] == 200" - "[BODY].status == UP" - "[RESPONSE_TIME] < 1000" - name: cat-fact - interval: 1m url: "https://cat-fact.herokuapp.com/facts/random" + interval: 1m conditions: - "[STATUS] == 200" - "[BODY].deleted == false" diff --git a/watchdog/alerting.go b/watchdog/alerting.go index 3952a15a..5ac4bafe 100644 --- a/watchdog/alerting.go +++ b/watchdog/alerting.go @@ -2,8 +2,6 @@ package watchdog import ( "encoding/json" - "fmt" - "github.com/TwinProduction/gatus/alerting/provider/custom" "github.com/TwinProduction/gatus/config" "github.com/TwinProduction/gatus/core" "log" @@ -36,44 +34,16 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf } continue } - var alertProvider *custom.AlertProvider - // TODO: leverage provider.AlertingProvider interface and config.GetAlertingProviderByAlertType(cfg, alert.Type) - // TODO: to support all types of alerts without having to explicitly define them here and in handleAlertsToResolve() - if alert.Type == core.SlackAlert { - if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() { - log.Printf("[watchdog][handleAlertsToTrigger] Sending Slack alert because alert with description='%s' has been triggered", alert.Description) - alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, false) - } else { - log.Printf("[watchdog][handleAlertsToTrigger] Not sending Slack alert despite being triggered, because there is no Slack webhook configured") - } - } else if alert.Type == core.PagerDutyAlert { - if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() { - log.Printf("[watchdog][handleAlertsToTrigger] Sending PagerDuty alert because alert with description='%s' has been triggered", alert.Description) - alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("trigger", "", service, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description)) - } else { - log.Printf("[watchdog][handleAlertsToTrigger] Not sending PagerDuty alert despite being triggered, because PagerDuty isn't configured properly") - } - } else if alert.Type == core.TwilioAlert { - if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() { - log.Printf("[watchdog][handleAlertsToTrigger] Sending Twilio alert because alert with description='%s' has been triggered", alert.Description) - alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description)) - } else { - log.Printf("[watchdog][handleAlertsToTrigger] Not sending Twilio alert despite being triggered, because Twilio config settings missing") - } - } else if alert.Type == core.CustomAlert { - if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() { - log.Printf("[watchdog][handleAlertsToTrigger] Sending custom alert because alert with description='%s' has been triggered", alert.Description) - alertProvider = cfg.Alerting.Custom - } else { - log.Printf("[watchdog][handleAlertsToTrigger] Not sending custom alert despite being triggered, because there is no custom url configured") - } - } - if alertProvider != nil { + alertProvider := config.GetAlertingProviderByAlertType(cfg, alert.Type) + if alertProvider != nil && alertProvider.IsValid() { + log.Printf("[watchdog][handleAlertsToTrigger] Sending %s alert because alert with description='%s' has been triggered", alert.Type, alert.Description) + customAlertProvider := alertProvider.ToCustomAlertProvider(service, alert, result, false) // TODO: retry on error var err error + // We need to extract the DedupKey from PagerDuty's response if alert.Type == core.PagerDutyAlert { var body []byte - body, err = alertProvider.Send(service.Name, alert.Description, true) + body, err = customAlertProvider.Send(service.Name, alert.Description, false) if err == nil { var response pagerDutyResponse err = json.Unmarshal(body, &response) @@ -84,13 +54,17 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf } } } else { - _, err = alertProvider.Send(service.Name, alert.Description, false) + // All other alert types don't need to extract anything from the body, so we can just send the request right away + _, err = customAlertProvider.Send(service.Name, alert.Description, false) } if err != nil { log.Printf("[watchdog][handleAlertsToTrigger] Ran into error sending an alert: %s", err.Error()) } else { alert.Triggered = true } + + } else { + log.Printf("[watchdog][handleAlertsToResolve] Not sending alert of type=%s despite being triggered, because the provider wasn't configured properly", alert.Type) } } } @@ -105,39 +79,12 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf if !alert.SendOnResolved { continue } - var alertProvider *custom.AlertProvider - if alert.Type == core.SlackAlert { - if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() { - log.Printf("[watchdog][handleAlertsToResolve] Sending Slack alert because alert with description='%s' has been resolved", alert.Description) - alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, true) - } else { - log.Printf("[watchdog][handleAlertsToResolve] Not sending Slack alert despite being resolved, because there is no Slack webhook configured") - } - } else if alert.Type == core.PagerDutyAlert { - if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() { - log.Printf("[watchdog][handleAlertsToResolve] Sending PagerDuty alert because alert with description='%s' has been resolved", alert.Description) - alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("resolve", alert.ResolveKey, service, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description)) - } else { - log.Printf("[watchdog][handleAlertsToResolve] Not sending PagerDuty alert despite being resolved, because PagerDuty isn't configured properly") - } - } else if alert.Type == core.TwilioAlert { - if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() { - log.Printf("[watchdog][handleAlertsToResolve] Sending Twilio alert because alert with description='%s' has been resolved", alert.Description) - alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description)) - } else { - log.Printf("[watchdog][handleAlertsToResolve] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly") - } - } else if alert.Type == core.CustomAlert { - if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() { - log.Printf("[watchdog][handleAlertsToResolve] Sending custom alert because alert with description='%s' has been resolved", alert.Description) - alertProvider = cfg.Alerting.Custom - } else { - log.Printf("[watchdog][handleAlertsToResolve] Not sending custom alert despite being resolved, because the custom provider isn't configured properly") - } - } - if alertProvider != nil { + alertProvider := config.GetAlertingProviderByAlertType(cfg, alert.Type) + if alertProvider != nil && alertProvider.IsValid() { + log.Printf("[watchdog][handleAlertsToResolve] Sending %s alert because alert with description='%s' has been resolved", alert.Type, alert.Description) + customAlertProvider := alertProvider.ToCustomAlertProvider(service, alert, result, true) // TODO: retry on error - _, err := alertProvider.Send(service.Name, alert.Description, true) + _, err := customAlertProvider.Send(service.Name, alert.Description, true) if err != nil { log.Printf("[watchdog][handleAlertsToResolve] Ran into error sending an alert: %s", err.Error()) } else { @@ -145,6 +92,8 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf alert.ResolveKey = "" } } + } else { + log.Printf("[watchdog][handleAlertsToResolve] Not sending alert of type=%s despite being resolved, because the provider wasn't configured properly", alert.Type) } } service.NumberOfFailuresInARow = 0