2019-09-05 01:37:13 +02:00
|
|
|
package watchdog
|
2019-09-07 03:59:50 +02:00
|
|
|
|
|
|
|
import (
|
2021-05-19 04:29:15 +02:00
|
|
|
"context"
|
2019-09-09 03:07:08 +02:00
|
|
|
"log"
|
2019-09-07 03:59:50 +02:00
|
|
|
"sync"
|
|
|
|
"time"
|
2020-10-30 16:30:03 +01:00
|
|
|
|
2022-12-06 07:41:09 +01:00
|
|
|
"github.com/TwiN/gatus/v5/alerting"
|
|
|
|
"github.com/TwiN/gatus/v5/config"
|
2023-05-03 04:41:22 +02:00
|
|
|
"github.com/TwiN/gatus/v5/config/connectivity"
|
2022-12-06 07:41:09 +01:00
|
|
|
"github.com/TwiN/gatus/v5/config/maintenance"
|
|
|
|
"github.com/TwiN/gatus/v5/core"
|
|
|
|
"github.com/TwiN/gatus/v5/metrics"
|
|
|
|
"github.com/TwiN/gatus/v5/storage/store"
|
2019-09-07 03:59:50 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
2021-10-23 22:47:12 +02:00
|
|
|
// monitoringMutex is used to prevent multiple endpoint from being evaluated at the same time.
|
2020-09-05 03:31:28 +02:00
|
|
|
// Without this, conditions using response time may become inaccurate.
|
|
|
|
monitoringMutex sync.Mutex
|
2021-05-19 04:29:15 +02:00
|
|
|
|
|
|
|
ctx context.Context
|
|
|
|
cancelFunc context.CancelFunc
|
2019-09-07 03:59:50 +02:00
|
|
|
)
|
|
|
|
|
2021-10-23 22:47:12 +02:00
|
|
|
// Monitor loops over each endpoint and starts a goroutine to monitor each endpoint separately
|
2019-12-04 22:44:35 +01:00
|
|
|
func Monitor(cfg *config.Config) {
|
2021-05-19 04:29:15 +02:00
|
|
|
ctx, cancelFunc = context.WithCancel(context.Background())
|
2021-10-23 22:47:12 +02:00
|
|
|
for _, endpoint := range cfg.Endpoints {
|
|
|
|
if endpoint.IsEnabled() {
|
2021-09-22 06:04:51 +02:00
|
|
|
// To prevent multiple requests from running at the same time, we'll wait for a little before each iteration
|
2021-10-29 01:35:46 +02:00
|
|
|
time.Sleep(777 * time.Millisecond)
|
2023-05-03 04:41:22 +02:00
|
|
|
go monitor(endpoint, cfg.Alerting, cfg.Maintenance, cfg.Connectivity, cfg.DisableMonitoringLock, cfg.Metrics, cfg.Debug, ctx)
|
2021-09-18 17:52:11 +02:00
|
|
|
}
|
2020-04-07 00:58:13 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-10-29 01:35:46 +02:00
|
|
|
// monitor a single endpoint in a loop
|
2023-05-03 04:41:22 +02:00
|
|
|
func monitor(endpoint *core.Endpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, connectivityConfig *connectivity.Config, disableMonitoringLock, enabledMetrics, debug bool, ctx context.Context) {
|
2021-05-19 04:29:15 +02:00
|
|
|
// Run it immediately on start
|
2023-05-03 04:41:22 +02:00
|
|
|
execute(endpoint, alertingConfig, maintenanceConfig, connectivityConfig, disableMonitoringLock, enabledMetrics, debug)
|
2021-05-19 04:29:15 +02:00
|
|
|
// Loop for the next executions
|
2020-04-07 00:58:13 +02:00
|
|
|
for {
|
2021-05-19 04:29:15 +02:00
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
2021-10-23 22:47:12 +02:00
|
|
|
log.Printf("[watchdog][monitor] Canceling current execution of group=%s; endpoint=%s", endpoint.Group, endpoint.Name)
|
2021-05-19 04:29:15 +02:00
|
|
|
return
|
2021-10-23 22:47:12 +02:00
|
|
|
case <-time.After(endpoint.Interval):
|
2023-05-03 04:41:22 +02:00
|
|
|
execute(endpoint, alertingConfig, maintenanceConfig, connectivityConfig, disableMonitoringLock, enabledMetrics, debug)
|
2020-09-05 03:31:28 +02:00
|
|
|
}
|
2021-05-19 04:29:15 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-05-03 04:41:22 +02:00
|
|
|
func execute(endpoint *core.Endpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, connectivityConfig *connectivity.Config, disableMonitoringLock, enabledMetrics, debug bool) {
|
2021-05-19 04:29:15 +02:00
|
|
|
if !disableMonitoringLock {
|
2021-10-23 22:47:12 +02:00
|
|
|
// By placing the lock here, we prevent multiple endpoints from being monitored at the exact same time, which
|
2021-05-19 04:29:15 +02:00
|
|
|
// could cause performance issues and return inaccurate results
|
|
|
|
monitoringMutex.Lock()
|
2023-05-03 04:41:22 +02:00
|
|
|
defer monitoringMutex.Unlock()
|
|
|
|
}
|
|
|
|
// If there's a connectivity checker configured, check if Gatus has internet connectivity
|
|
|
|
if connectivityConfig != nil && connectivityConfig.Checker != nil && !connectivityConfig.Checker.IsConnected() {
|
|
|
|
log.Println("[watchdog][execute] No connectivity; skipping execution")
|
|
|
|
return
|
2021-05-19 04:29:15 +02:00
|
|
|
}
|
|
|
|
if debug {
|
2021-10-23 22:47:12 +02:00
|
|
|
log.Printf("[watchdog][execute] Monitoring group=%s; endpoint=%s", endpoint.Group, endpoint.Name)
|
2021-05-19 04:29:15 +02:00
|
|
|
}
|
2021-10-23 22:47:12 +02:00
|
|
|
result := endpoint.EvaluateHealth()
|
2021-05-19 04:29:15 +02:00
|
|
|
if enabledMetrics {
|
2022-06-17 00:55:51 +02:00
|
|
|
metrics.PublishMetricsForEndpoint(endpoint, result)
|
2021-05-19 04:29:15 +02:00
|
|
|
}
|
2021-10-23 22:47:12 +02:00
|
|
|
UpdateEndpointStatuses(endpoint, result)
|
2023-03-15 01:02:31 +01:00
|
|
|
if debug && !result.Success {
|
|
|
|
log.Printf("[watchdog][execute] Monitored group=%s; endpoint=%s; success=%v; errors=%d; duration=%s; body=%s", endpoint.Group, endpoint.Name, result.Success, len(result.Errors), result.Duration.Round(time.Millisecond), result.Body)
|
|
|
|
} else {
|
|
|
|
log.Printf("[watchdog][execute] Monitored group=%s; endpoint=%s; success=%v; errors=%d; duration=%s", endpoint.Group, endpoint.Name, result.Success, len(result.Errors), result.Duration.Round(time.Millisecond))
|
|
|
|
}
|
2021-09-22 06:04:51 +02:00
|
|
|
if !maintenanceConfig.IsUnderMaintenance() {
|
2021-12-03 03:05:17 +01:00
|
|
|
// TODO: Consider moving this after the monitoring lock is unlocked? I mean, how much noise can a single alerting provider cause...
|
2021-10-23 22:47:12 +02:00
|
|
|
HandleAlerting(endpoint, result, alertingConfig, debug)
|
2021-09-22 06:04:51 +02:00
|
|
|
} else if debug {
|
|
|
|
log.Println("[watchdog][execute] Not handling alerting because currently in the maintenance window")
|
|
|
|
}
|
2021-05-19 04:29:15 +02:00
|
|
|
if debug {
|
2021-10-23 22:47:12 +02:00
|
|
|
log.Printf("[watchdog][execute] Waiting for interval=%s before monitoring group=%s endpoint=%s again", endpoint.Interval, endpoint.Group, endpoint.Name)
|
2021-05-19 04:29:15 +02:00
|
|
|
}
|
2020-09-05 00:23:56 +02:00
|
|
|
}
|
2020-11-27 00:09:01 +01:00
|
|
|
|
2021-10-23 22:47:12 +02:00
|
|
|
// UpdateEndpointStatuses updates the slice of endpoint statuses
|
|
|
|
func UpdateEndpointStatuses(endpoint *core.Endpoint, result *core.Result) {
|
2021-10-29 01:35:46 +02:00
|
|
|
if err := store.Get().Insert(endpoint, result); err != nil {
|
2021-10-23 22:47:12 +02:00
|
|
|
log.Println("[watchdog][UpdateEndpointStatuses] Failed to insert data in storage:", err.Error())
|
2021-09-22 06:04:51 +02:00
|
|
|
}
|
2020-11-27 00:09:01 +01:00
|
|
|
}
|
2021-05-19 04:29:15 +02:00
|
|
|
|
2021-10-23 22:47:12 +02:00
|
|
|
// Shutdown stops monitoring all endpoints
|
2023-08-05 00:30:15 +02:00
|
|
|
func Shutdown(cfg *config.Config) {
|
|
|
|
// Disable all the old HTTP connections
|
|
|
|
for _, endpoint := range cfg.Endpoints {
|
|
|
|
endpoint.Close()
|
|
|
|
}
|
2021-05-19 04:29:15 +02:00
|
|
|
cancelFunc()
|
|
|
|
}
|