Start working on #12: Fix inconsistencies in alerting providers

README.md

@@ -94,15 +94,17 @@ Note that you can also add environment variables in the configuration file (i.e.
 | `services[].alerts[].send-on-resolved`   | Whether to send a notification once a triggered alert is marked as resolved   | `false`        |
 | `services[].alerts[].description`        | Description of the alert. Will be included in the alert sent                  | `""`           |
 | `alerting`                               | Configuration for alerting                                                    | `{}`           |
-| `alerting.slack`                         | Webhook to use for alerts of type `slack`                                     | `""`           |
+| `alerting.slack`                         | Configuration for alerts of type `slack`                                      | `""`           |
-| `alerting.pagerduty`                     | PagerDuty Events API v2 integration key. Used for alerts of type `pagerduty`  | `""`           |
+| `alerting.slack.webhook-url`             | Slack Webhook URL                                                             | Required `""`  |
+| `alerting.pagerduty`                     | Configuration for alerts of type `pagerduty`                                  | `""`           |
+| `alerting.pagerduty.integration-key`     | PagerDuty Events API v2 integration key.                                      | Required `""`  |
 | `alerting.twilio`                        | Settings for alerts of type `twilio`                                          | `""`           |
 | `alerting.twilio.sid`                    | Twilio account SID                                                            | Required `""`  |
 | `alerting.twilio.token`                  | Twilio auth token                                                             | Required `""`  |
 | `alerting.twilio.from`                   | Number to send Twilio alerts from                                             | Required `""`  |
 | `alerting.twilio.to`                     | Number to send twilio alerts to                                               | Required `""`  |
 | `alerting.custom`                        | Configuration for custom actions on failure or alerts                         | `""`           |
-| `alerting.custom.url`                    | Custom alerting request url                                                   | `""`           |
+| `alerting.custom.url`                    | Custom alerting request url                                                   | Required `""`  |
 | `alerting.custom.body`                   | Custom alerting request body.                                                 | `""`           |
 | `alerting.custom.headers`                | Custom alerting request headers                                               | `{}`           |
 
@@ -133,7 +135,8 @@ Here are some examples of conditions you can use:
 
 ```yaml
 alerting:
-  slack: "https://hooks.slack.com/services/**********/**********/**********"
+  slack: 
+    webhook-url: "https://hooks.slack.com/services/**********/**********/**********"
 services:
   - name: twinnation
     interval: 30s
@@ -168,7 +171,8 @@ PagerDuty instead.
 
 ```yaml
 alerting:
-  pagerduty: "********************************"
+  pagerduty: 
+    integration-key: "********************************"
 services:
   - name: twinnation
     interval: 30s

alerting/config.go

@@ -0,0 +1,8 @@
+package alerting
+
+type Config struct {
+	Slack     *SlackAlertProvider     `yaml:"slack"`
+	PagerDuty *PagerDutyAlertProvider `yaml:"pagerduty"`
+	Twilio    *TwilioAlertProvider    `yaml:"twilio"`
+	Custom    *CustomAlertProvider    `yaml:"custom"`
+}

alerting/custom.go

@@ -0,0 +1,80 @@
+package alerting
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/TwinProduction/gatus/client"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+type CustomAlertProvider struct {
+	Url     string            `yaml:"url"`
+	Method  string            `yaml:"method,omitempty"`
+	Body    string            `yaml:"body,omitempty"`
+	Headers map[string]string `yaml:"headers,omitempty"`
+}
+
+func (provider *CustomAlertProvider) IsValid() bool {
+	return len(provider.Url) > 0
+}
+
+func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request {
+	body := provider.Body
+	providerUrl := provider.Url
+	method := provider.Method
+	if strings.Contains(body, "[ALERT_DESCRIPTION]") {
+		body = strings.ReplaceAll(body, "[ALERT_DESCRIPTION]", alertDescription)
+	}
+	if strings.Contains(body, "[SERVICE_NAME]") {
+		body = strings.ReplaceAll(body, "[SERVICE_NAME]", serviceName)
+	}
+	if strings.Contains(body, "[ALERT_TRIGGERED_OR_RESOLVED]") {
+		if resolved {
+			body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
+		} else {
+			body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
+		}
+	}
+	if strings.Contains(providerUrl, "[ALERT_DESCRIPTION]") {
+		providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_DESCRIPTION]", alertDescription)
+	}
+	if strings.Contains(providerUrl, "[SERVICE_NAME]") {
+		providerUrl = strings.ReplaceAll(providerUrl, "[SERVICE_NAME]", serviceName)
+	}
+	if strings.Contains(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]") {
+		if resolved {
+			providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
+		} else {
+			providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
+		}
+	}
+	if len(method) == 0 {
+		method = "GET"
+	}
+	bodyBuffer := bytes.NewBuffer([]byte(body))
+	request, _ := http.NewRequest(method, providerUrl, bodyBuffer)
+	for k, v := range provider.Headers {
+		request.Header.Set(k, v)
+	}
+	return request
+}
+
+// Send a request to the alert provider and return the body
+func (provider *CustomAlertProvider) Send(serviceName, alertDescription string, resolved bool) ([]byte, error) {
+	request := provider.buildRequest(serviceName, alertDescription, resolved)
+	response, err := client.GetHttpClient().Do(request)
+	if err != nil {
+		return nil, err
+	}
+	if response.StatusCode > 399 {
+		body, err := ioutil.ReadAll(response.Body)
+		if err != nil {
+			return nil, fmt.Errorf("call to provider alert returned status code %d", response.StatusCode)
+		} else {
+			return nil, fmt.Errorf("call to provider alert returned status code %d: %s", response.StatusCode, string(body))
+		}
+	}
+	return ioutil.ReadAll(response.Body)
+}

alerting/custom_test.go

@@ -1,10 +1,21 @@
-package core
+package alerting
 
 import (
 	"io/ioutil"
 	"testing"
 )
 
+func TestCustomAlertProvider_IsValid(t *testing.T) {
+	invalidProvider := CustomAlertProvider{Url: ""}
+	if invalidProvider.IsValid() {
+		t.Error("provider shouldn't have been valid")
+	}
+	validProvider := CustomAlertProvider{Url: "http://example.com"}
+	if !validProvider.IsValid() {
+		t.Error("provider should've been valid")
+	}
+}
+
 func TestCustomAlertProvider_buildRequestWhenResolved(t *testing.T) {
 	const (
 		ExpectedUrl  = "http://example.com/service-name"

alerting/pagerduty.go

@@ -1,7 +1,35 @@
 package alerting
 
-type pagerDutyResponse struct {
+import (
-	Status   string `json:"status"`
+	"fmt"
-	Message  string `json:"message"`
+	"github.com/TwinProduction/gatus/core"
-	DedupKey string `json:"dedup_key"`
+)
+
+type PagerDutyAlertProvider struct {
+	IntegrationKey string `yaml:"integration-key"`
+}
+
+func (provider *PagerDutyAlertProvider) IsValid() bool {
+	return len(provider.IntegrationKey) == 32
+}
+
+// https://developer.pagerduty.com/docs/events-api-v2/trigger-events/
+func (provider *PagerDutyAlertProvider) ToCustomAlertProvider(eventAction, resolveKey string, service *core.Service, message string) *CustomAlertProvider {
+	return &CustomAlertProvider{
+		Url:    "https://events.pagerduty.com/v2/enqueue",
+		Method: "POST",
+		Body: fmt.Sprintf(`{
+  "routing_key": "%s",
+  "dedup_key": "%s",
+  "event_action": "%s",
+  "payload": {
+    "summary": "%s",
+    "source": "%s",
+    "severity": "critical"
+  }
+}`, provider.IntegrationKey, resolveKey, eventAction, message, service.Name),
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+	}
 }

alerting/pagerduty_test.go

@@ -0,0 +1,14 @@
+package alerting
+
+import "testing"
+
+func TestPagerDutyAlertProvider_IsValid(t *testing.T) {
+	invalidProvider := PagerDutyAlertProvider{IntegrationKey: ""}
+	if invalidProvider.IsValid() {
+		t.Error("provider shouldn't have been valid")
+	}
+	validProvider := PagerDutyAlertProvider{IntegrationKey: "00000000000000000000000000000000"}
+	if !validProvider.IsValid() {
+		t.Error("provider should've been valid")
+	}
+}

alerting/slack.go

@@ -0,0 +1,59 @@
+package alerting
+
+import (
+	"fmt"
+	"github.com/TwinProduction/gatus/core"
+)
+
+type SlackAlertProvider struct {
+	WebhookUrl string `yaml:"webhook-url"`
+}
+
+func (provider *SlackAlertProvider) IsValid() bool {
+	return len(provider.WebhookUrl) > 0
+}
+
+func (provider *SlackAlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *CustomAlertProvider {
+	var message string
+	var color string
+	if resolved {
+		message = fmt.Sprintf("An alert for *%s* has been resolved after passing successfully %d time(s) in a row", service.Name, alert.SuccessThreshold)
+		color = "#36A64F"
+	} else {
+		message = fmt.Sprintf("An alert for *%s* has been triggered due to having failed %d time(s) in a row", service.Name, alert.FailureThreshold)
+		color = "#DD0000"
+	}
+	var results string
+	for _, conditionResult := range result.ConditionResults {
+		var prefix string
+		if conditionResult.Success {
+			prefix = ":heavy_check_mark:"
+		} else {
+			prefix = ":x:"
+		}
+		results += fmt.Sprintf("%s - `%s`\n", prefix, conditionResult.Condition)
+	}
+	return &CustomAlertProvider{
+		Url:    provider.WebhookUrl,
+		Method: "POST",
+		Body: fmt.Sprintf(`{
+  "text": "",
+  "attachments": [
+    {
+      "title": ":helmet_with_white_cross: Gatus",
+      "text": "%s:\n> %s",
+      "short": false,
+      "color": "%s",
+      "fields": [
+        {
+          "title": "Condition results",
+          "value": "%s",
+          "short": false
+        }
+      ]
+    },
+  ]
+}`, message, alert.Description, color, results),
+		Headers: map[string]string{"Content-Type": "application/json"},
+	}
+}

alerting/slack_test.go

@@ -0,0 +1,14 @@
+package alerting
+
+import "testing"
+
+func TestSlackAlertProvider_IsValid(t *testing.T) {
+	invalidProvider := SlackAlertProvider{WebhookUrl: ""}
+	if invalidProvider.IsValid() {
+		t.Error("provider shouldn't have been valid")
+	}
+	validProvider := SlackAlertProvider{WebhookUrl: "http://example.com"}
+	if !validProvider.IsValid() {
+		t.Error("provider should've been valid")
+	}
+}

alerting/twilio.go

@@ -0,0 +1,34 @@
+package alerting
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/url"
+)
+
+type TwilioAlertProvider struct {
+	SID   string `yaml:"sid"`
+	Token string `yaml:"token"`
+	From  string `yaml:"from"`
+	To    string `yaml:"to"`
+}
+
+func (provider *TwilioAlertProvider) IsValid() bool {
+	return len(provider.Token) > 0 && len(provider.SID) > 0 && len(provider.From) > 0 && len(provider.To) > 0
+}
+
+func (provider *TwilioAlertProvider) ToCustomAlertProvider(message string) *CustomAlertProvider {
+	return &CustomAlertProvider{
+		Url:    fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID),
+		Method: "POST",
+		Body: url.Values{
+			"To":   {provider.To},
+			"From": {provider.From},
+			"Body": {message},
+		}.Encode(),
+		Headers: map[string]string{
+			"Content-Type":  "application/x-www-form-urlencoded",
+			"Authorization": fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", provider.SID, provider.Token)))),
+		},
+	}
+}

alerting/twilio_test.go

@@ -0,0 +1,19 @@
+package alerting
+
+import "testing"
+
+func TestTwilioAlertProvider_IsValid(t *testing.T) {
+	invalidProvider := TwilioAlertProvider{}
+	if invalidProvider.IsValid() {
+		t.Error("provider shouldn't have been valid")
+	}
+	validProvider := TwilioAlertProvider{
+		SID:   "1",
+		Token: "1",
+		From:  "1",
+		To:    "1",
+	}
+	if !validProvider.IsValid() {
+		t.Error("provider should've been valid")
+	}
+}

config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"errors"
+	"github.com/TwinProduction/gatus/alerting"
 	"github.com/TwinProduction/gatus/core"
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
@@ -23,7 +24,7 @@ var (
 type Config struct {
 	Metrics  bool             `yaml:"metrics"`
 	Debug    bool             `yaml:"debug"`
-	Alerting *core.AlertingConfig `yaml:"alerting"`
+	Alerting *alerting.Config `yaml:"alerting"`
 	Services []*core.Service  `yaml:"services"`
 }
 

config/config_test.go

@@ -121,7 +121,8 @@ badconfig:
 func TestParseAndValidateConfigBytesWithAlerting(t *testing.T) {
 	config, err := parseAndValidateConfigBytes([]byte(`
 alerting:
-  slack: "http://example.com"
+  slack:
+    webhook-url: "http://example.com"
 services:
   - name: twinnation
     url: https://twinnation.org/actuator/health
@@ -145,7 +146,7 @@ services:
 	if config.Alerting == nil {
 		t.Fatal("config.AlertingConfig shouldn't have been nil")
 	}
-	if config.Alerting.Slack != "http://example.com" {
+	if config.Alerting.Slack.WebhookUrl != "http://example.com" {
 		t.Errorf("Slack webhook should've been %s, but was %s", "http://example.com", config.Alerting.Slack)
 	}
 	if len(config.Services) != 1 {

core/alerting.go

@@ -1,178 +0,0 @@
-package core
-
-import (
-	"bytes"
-	"encoding/base64"
-	"fmt"
-	"github.com/TwinProduction/gatus/client"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"strings"
-)
-
-type AlertingConfig struct {
-	Slack     string               `yaml:"slack"`
-	PagerDuty string               `yaml:"pagerduty"`
-	Twilio    *TwilioAlertProvider `yaml:"twilio"`
-	Custom    *CustomAlertProvider `yaml:"custom"`
-}
-
-type TwilioAlertProvider struct {
-	SID   string `yaml:"sid"`
-	Token string `yaml:"token"`
-	From  string `yaml:"from"`
-	To    string `yaml:"to"`
-}
-
-func (provider *TwilioAlertProvider) IsValid() bool {
-	return len(provider.Token) > 0 && len(provider.SID) > 0 && len(provider.From) > 0 && len(provider.To) > 0
-}
-
-type CustomAlertProvider struct {
-	Url     string            `yaml:"url"`
-	Method  string            `yaml:"method,omitempty"`
-	Body    string            `yaml:"body,omitempty"`
-	Headers map[string]string `yaml:"headers,omitempty"`
-}
-
-func (provider *CustomAlertProvider) IsValid() bool {
-	return len(provider.Url) > 0
-}
-
-func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request {
-	body := provider.Body
-	providerUrl := provider.Url
-	if strings.Contains(body, "[ALERT_DESCRIPTION]") {
-		body = strings.ReplaceAll(body, "[ALERT_DESCRIPTION]", alertDescription)
-	}
-	if strings.Contains(body, "[SERVICE_NAME]") {
-		body = strings.ReplaceAll(body, "[SERVICE_NAME]", serviceName)
-	}
-	if strings.Contains(body, "[ALERT_TRIGGERED_OR_RESOLVED]") {
-		if resolved {
-			body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
-		} else {
-			body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
-		}
-	}
-	if strings.Contains(providerUrl, "[ALERT_DESCRIPTION]") {
-		providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_DESCRIPTION]", alertDescription)
-	}
-	if strings.Contains(providerUrl, "[SERVICE_NAME]") {
-		providerUrl = strings.ReplaceAll(providerUrl, "[SERVICE_NAME]", serviceName)
-	}
-	if strings.Contains(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]") {
-		if resolved {
-			providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
-		} else {
-			providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
-		}
-	}
-	bodyBuffer := bytes.NewBuffer([]byte(body))
-	request, _ := http.NewRequest(provider.Method, providerUrl, bodyBuffer)
-	for k, v := range provider.Headers {
-		request.Header.Set(k, v)
-	}
-	return request
-}
-
-// Send a request to the alert provider and return the body
-func (provider *CustomAlertProvider) Send(serviceName, alertDescription string, resolved bool) ([]byte, error) {
-	request := provider.buildRequest(serviceName, alertDescription, resolved)
-	response, err := client.GetHttpClient().Do(request)
-	if err != nil {
-		return nil, err
-	}
-	if response.StatusCode > 399 {
-		body, err := ioutil.ReadAll(response.Body)
-		if err != nil {
-			return nil, fmt.Errorf("call to provider alert returned status code %d", response.StatusCode)
-		} else {
-			return nil, fmt.Errorf("call to provider alert returned status code %d: %s", response.StatusCode, string(body))
-		}
-	}
-	return ioutil.ReadAll(response.Body)
-}
-
-func CreateSlackCustomAlertProvider(slackWebHookUrl string, service *Service, alert *Alert, result *Result, resolved bool) *CustomAlertProvider {
-	var message string
-	var color string
-	if resolved {
-		message = fmt.Sprintf("An alert for *%s* has been resolved after passing successfully %d time(s) in a row", service.Name, alert.SuccessThreshold)
-		color = "#36A64F"
-	} else {
-		message = fmt.Sprintf("An alert for *%s* has been triggered due to having failed %d time(s) in a row", service.Name, alert.FailureThreshold)
-		color = "#DD0000"
-	}
-	var results string
-	for _, conditionResult := range result.ConditionResults {
-		var prefix string
-		if conditionResult.Success {
-			prefix = ":heavy_check_mark:"
-		} else {
-			prefix = ":x:"
-		}
-		results += fmt.Sprintf("%s - `%s`\n", prefix, conditionResult.Condition)
-	}
-	return &CustomAlertProvider{
-		Url:    slackWebHookUrl,
-		Method: "POST",
-		Body: fmt.Sprintf(`{
-  "text": "",
-  "attachments": [
-    {
-      "title": ":helmet_with_white_cross: Gatus",
-      "text": "%s:\n> %s",
-      "short": false,
-      "color": "%s",
-      "fields": [
-        {
-          "title": "Condition results",
-          "value": "%s",
-          "short": false
-        }
-      ]
-    },
-  ]
-}`, message, alert.Description, color, results),
-		Headers: map[string]string{"Content-Type": "application/json"},
-	}
-}
-
-func CreateTwilioCustomAlertProvider(provider *TwilioAlertProvider, message string) *CustomAlertProvider {
-	return &CustomAlertProvider{
-		Url:    fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID),
-		Method: "POST",
-		Body: url.Values{
-			"To":   {provider.To},
-			"From": {provider.From},
-			"Body": {message},
-		}.Encode(),
-		Headers: map[string]string{
-			"Content-Type":  "application/x-www-form-urlencoded",
-			"Authorization": fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", provider.SID, provider.Token)))),
-		},
-	}
-}
-
-// https://developer.pagerduty.com/docs/events-api-v2/trigger-events/
-func CreatePagerDutyCustomAlertProvider(routingKey, eventAction, resolveKey string, service *Service, message string) *CustomAlertProvider {
-	return &CustomAlertProvider{
-		Url:    "https://events.pagerduty.com/v2/enqueue",
-		Method: "POST",
-		Body: fmt.Sprintf(`{
-  "routing_key": "%s",
-  "dedup_key": "%s",
-  "event_action": "%s",
-  "payload": {
-    "summary": "%s",
-    "source": "%s",
-    "severity": "critical"
-  }
-}`, routingKey, resolveKey, eventAction, message, service.Name),
-		Headers: map[string]string{
-			"Content-Type": "application/json",
-		},
-	}
-}

docs/pagerduty-integration-guide.md

@@ -30,10 +30,11 @@ If you need help with this integration, please create an issue at https://github
 
 
 ## In Gatus
-In your configuration file, you must first specify the integration key in `alerting.pagerduty`, like so:
+In your configuration file, you must first specify the integration key at `alerting.pagerduty.integration-key`, like so:
 ```yaml
 alerting:
-  pagerduty: "********************************"
+  pagerduty: 
+    integration-key: "********************************"
 ```
 You can now add alerts of type `pagerduty` in the services you've defined, like so:
 ```yaml

watchdog/alerting.go

@@ -1,15 +1,16 @@
-package alerting
+package watchdog
 
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/TwinProduction/gatus/alerting"
 	"github.com/TwinProduction/gatus/config"
 	"github.com/TwinProduction/gatus/core"
 	"log"
 )
 
-// Handle takes care of alerts to resolve and alerts to trigger based on result success or failure
+// HandleAlerting takes care of alerts to resolve and alerts to trigger based on result success or failure
-func Handle(service *core.Service, result *core.Result) {
+func HandleAlerting(service *core.Service, result *core.Result) {
 	cfg := config.Get()
 	if cfg.Alerting == nil {
 		return
@@ -31,43 +32,38 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf
 		}
 		if alert.Triggered {
 			if cfg.Debug {
-				log.Printf("[alerting][handleAlertsToTrigger] Alert with description='%s' has already been triggered, skipping", alert.Description)
+				log.Printf("[watchdog][handleAlertsToTrigger] Alert with description='%s' has already been triggered, skipping", alert.Description)
 			}
 			continue
 		}
-		var alertProvider *core.CustomAlertProvider
+		var alertProvider *alerting.CustomAlertProvider
 		if alert.Type == core.SlackAlert {
-			if len(cfg.Alerting.Slack) > 0 {
+			if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() {
-				log.Printf("[alerting][handleAlertsToTrigger] Sending Slack alert because alert with description='%s' has been triggered", alert.Description)
+				log.Printf("[watchdog][handleAlertsToTrigger] Sending Slack alert because alert with description='%s' has been triggered", alert.Description)
-				alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, false)
+				alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, false)
 			} else {
-				log.Printf("[alerting][handleAlertsToTrigger] Not sending Slack alert despite being triggered, because there is no Slack webhook configured")
+				log.Printf("[watchdog][handleAlertsToTrigger] Not sending Slack alert despite being triggered, because there is no Slack webhook configured")
 			}
 		} else if alert.Type == core.PagerDutyAlert {
-			if len(cfg.Alerting.PagerDuty) > 0 {
+			if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() {
-				log.Printf("[alerting][handleAlertsToTrigger] Sending PagerDuty alert because alert with description='%s' has been triggered", alert.Description)
+				log.Printf("[watchdog][handleAlertsToTrigger] Sending PagerDuty alert because alert with description='%s' has been triggered", alert.Description)
-				alertProvider = core.CreatePagerDutyCustomAlertProvider(cfg.Alerting.PagerDuty, "trigger", "", service, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
+				alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("trigger", "", service, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
 			} else {
-				log.Printf("[alerting][handleAlertsToTrigger] Not sending PagerDuty alert despite being triggered, because PagerDuty isn't configured properly")
+				log.Printf("[watchdog][handleAlertsToTrigger] Not sending PagerDuty alert despite being triggered, because PagerDuty isn't configured properly")
 			}
 		} else if alert.Type == core.TwilioAlert {
 			if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
-				log.Printf("[alerting][handleAlertsToTrigger] Sending Twilio alert because alert with description='%s' has been triggered", alert.Description)
+				log.Printf("[watchdog][handleAlertsToTrigger] Sending Twilio alert because alert with description='%s' has been triggered", alert.Description)
-				alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
+				alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
 			} else {
-				log.Printf("[alerting][handleAlertsToTrigger] Not sending Twilio alert despite being triggered, because Twilio config settings missing")
+				log.Printf("[watchdog][handleAlertsToTrigger] Not sending Twilio alert despite being triggered, because Twilio config settings missing")
 			}
 		} else if alert.Type == core.CustomAlert {
 			if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
-				log.Printf("[alerting][handleAlertsToTrigger] Sending custom alert because alert with description='%s' has been triggered", alert.Description)
+				log.Printf("[watchdog][handleAlertsToTrigger] Sending custom alert because alert with description='%s' has been triggered", alert.Description)
-				alertProvider = &core.CustomAlertProvider{
+				alertProvider = cfg.Alerting.Custom
-					Url:     cfg.Alerting.Custom.Url,
-					Method:  cfg.Alerting.Custom.Method,
-					Body:    cfg.Alerting.Custom.Body,
-					Headers: cfg.Alerting.Custom.Headers,
-				}
 			} else {
-				log.Printf("[alerting][handleAlertsToTrigger] Not sending custom alert despite being triggered, because there is no custom url configured")
+				log.Printf("[watchdog][handleAlertsToTrigger] Not sending custom alert despite being triggered, because there is no custom url configured")
 			}
 		}
 		if alertProvider != nil {
@@ -80,7 +76,7 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf
 					var response pagerDutyResponse
 					err = json.Unmarshal(body, &response)
 					if err != nil {
-						log.Printf("[alerting][handleAlertsToTrigger] Ran into error unmarshaling pager duty response: %s", err.Error())
+						log.Printf("[watchdog][handleAlertsToTrigger] Ran into error unmarshaling pager duty response: %s", err.Error())
 					} else {
 						alert.ResolveKey = response.DedupKey
 					}
@@ -89,7 +85,7 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf
 				_, err = alertProvider.Send(service.Name, alert.Description, false)
 			}
 			if err != nil {
-				log.Printf("[alerting][handleAlertsToTrigger] Ran into error sending an alert: %s", err.Error())
+				log.Printf("[watchdog][handleAlertsToTrigger] Ran into error sending an alert: %s", err.Error())
 			} else {
 				alert.Triggered = true
 			}
@@ -107,46 +103,46 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf
 		if !alert.SendOnResolved {
 			continue
 		}
-		var alertProvider *core.CustomAlertProvider
+		var alertProvider *alerting.CustomAlertProvider
 		if alert.Type == core.SlackAlert {
-			if len(cfg.Alerting.Slack) > 0 {
+			if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() {
-				log.Printf("[alerting][handleAlertsToResolve] Sending Slack alert because alert with description='%s' has been resolved", alert.Description)
+				log.Printf("[watchdog][handleAlertsToResolve] Sending Slack alert because alert with description='%s' has been resolved", alert.Description)
-				alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, true)
+				alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, true)
 			} else {
-				log.Printf("[alerting][handleAlertsToResolve] Not sending Slack alert despite being resolved, because there is no Slack webhook configured")
+				log.Printf("[watchdog][handleAlertsToResolve] Not sending Slack alert despite being resolved, because there is no Slack webhook configured")
 			}
 		} else if alert.Type == core.PagerDutyAlert {
-			if len(cfg.Alerting.PagerDuty) > 0 {
+			if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() {
-				log.Printf("[alerting][handleAlertsToResolve] Sending PagerDuty alert because alert with description='%s' has been resolved", alert.Description)
+				log.Printf("[watchdog][handleAlertsToResolve] Sending PagerDuty alert because alert with description='%s' has been resolved", alert.Description)
-				alertProvider = core.CreatePagerDutyCustomAlertProvider(cfg.Alerting.PagerDuty, "resolve", alert.ResolveKey, service, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
+				alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("resolve", alert.ResolveKey, service, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
 			} else {
-				log.Printf("[alerting][handleAlertsToResolve] Not sending PagerDuty alert despite being resolved, because PagerDuty isn't configured properly")
+				log.Printf("[watchdog][handleAlertsToResolve] Not sending PagerDuty alert despite being resolved, because PagerDuty isn't configured properly")
 			}
 		} else if alert.Type == core.TwilioAlert {
 			if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
-				log.Printf("[alerting][handleAlertsToResolve] Sending Twilio alert because alert with description='%s' has been resolved", alert.Description)
+				log.Printf("[watchdog][handleAlertsToResolve] Sending Twilio alert because alert with description='%s' has been resolved", alert.Description)
-				alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
+				alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
 			} else {
-				log.Printf("[alerting][handleAlertsToResolve] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly")
+				log.Printf("[watchdog][handleAlertsToResolve] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly")
 			}
 		} else if alert.Type == core.CustomAlert {
 			if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
-				log.Printf("[alerting][handleAlertsToResolve] Sending custom alert because alert with description='%s' has been resolved", alert.Description)
+				log.Printf("[watchdog][handleAlertsToResolve] Sending custom alert because alert with description='%s' has been resolved", alert.Description)
-				alertProvider = &core.CustomAlertProvider{
+				alertProvider = &alerting.CustomAlertProvider{
 					Url:     cfg.Alerting.Custom.Url,
 					Method:  cfg.Alerting.Custom.Method,
 					Body:    cfg.Alerting.Custom.Body,
 					Headers: cfg.Alerting.Custom.Headers,
 				}
 			} else {
-				log.Printf("[alerting][handleAlertsToResolve] Not sending custom alert despite being resolved, because the custom provider isn't configured properly")
+				log.Printf("[watchdog][handleAlertsToResolve] Not sending custom alert despite being resolved, because the custom provider isn't configured properly")
 			}
 		}
 		if alertProvider != nil {
 			// TODO: retry on error
 			_, err := alertProvider.Send(service.Name, alert.Description, true)
 			if err != nil {
-				log.Printf("[alerting][handleAlertsToResolve] Ran into error sending an alert: %s", err.Error())
+				log.Printf("[watchdog][handleAlertsToResolve] Ran into error sending an alert: %s", err.Error())
 			} else {
 				if alert.Type == core.PagerDutyAlert {
 					alert.ResolveKey = ""
@@ -156,3 +152,9 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf
 	}
 	service.NumberOfFailuresInARow = 0
 }
+
+type pagerDutyResponse struct {
+	Status   string `json:"status"`
+	Message  string `json:"message"`
+	DedupKey string `json:"dedup_key"`
+}

watchdog/watchdog.go

@@ -3,7 +3,6 @@ package watchdog
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/TwinProduction/gatus/alerting"
 	"github.com/TwinProduction/gatus/config"
 	"github.com/TwinProduction/gatus/core"
 	"github.com/TwinProduction/gatus/metric"
@@ -71,7 +70,7 @@ func monitor(service *core.Service) {
 			result.Duration.Round(time.Millisecond),
 			extra,
 		)
-		alerting.Handle(service, result)
+		HandleAlerting(service, result)
 		if cfg.Debug {
 			log.Printf("[watchdog][monitor] Waiting for interval=%s before monitoring serviceName=%s again", service.Interval, service.Name)
 		}