gatus/watchdog/watchdog.go

package watchdog

import (
	"context"
	"log"
	"sync"
	"time"

	"github.com/TwinProduction/gatus/alerting"
	"github.com/TwinProduction/gatus/config"
	"github.com/TwinProduction/gatus/core"
	"github.com/TwinProduction/gatus/metric"
	"github.com/TwinProduction/gatus/storage"
)

var (
	// monitoringMutex is used to prevent multiple services from being evaluated at the same time.
	// Without this, conditions using response time may become inaccurate.
	monitoringMutex sync.Mutex

	ctx        context.Context
	cancelFunc context.CancelFunc
)

// Monitor loops over each services and starts a goroutine to monitor each services separately
func Monitor(cfg *config.Config) {
	ctx, cancelFunc = context.WithCancel(context.Background())
	for _, service := range cfg.Services {
		// To prevent multiple requests from running at the same time, we'll wait for a little bit before each iteration
		time.Sleep(1111 * time.Millisecond)
		go monitor(service, cfg.Alerting, cfg.DisableMonitoringLock, cfg.Metrics, cfg.Debug, ctx)
	}
}

// monitor monitors a single service in a loop
func monitor(service *core.Service, alertingConfig *alerting.Config, disableMonitoringLock, enabledMetrics, debug bool, ctx context.Context) {
	// Run it immediately on start
	execute(service, alertingConfig, disableMonitoringLock, enabledMetrics, debug)
	// Loop for the next executions
	for {
		select {
		case <-ctx.Done():
			log.Printf("[watchdog][monitor] Canceling current execution of group=%s; service=%s", service.Group, service.Name)
			return
		case <-time.After(service.Interval):
			execute(service, alertingConfig, disableMonitoringLock, enabledMetrics, debug)
		}
	}
}

func execute(service *core.Service, alertingConfig *alerting.Config, disableMonitoringLock, enabledMetrics, debug bool) {
	if !disableMonitoringLock {
		// By placing the lock here, we prevent multiple services from being monitored at the exact same time, which
		// could cause performance issues and return inaccurate results
		monitoringMutex.Lock()
	}
	if debug {
		log.Printf("[watchdog][execute] Monitoring group=%s; service=%s", service.Group, service.Name)
	}
	result := service.EvaluateHealth()
	if enabledMetrics {
		metric.PublishMetricsForService(service, result)
	}
	UpdateServiceStatuses(service, result)
	log.Printf(
		"[watchdog][execute] Monitored group=%s; service=%s; success=%v; errors=%d; duration=%s",
		service.Group,
		service.Name,
		result.Success,
		len(result.Errors),
		result.Duration.Round(time.Millisecond),
	)
	HandleAlerting(service, result, alertingConfig, debug)
	if debug {
		log.Printf("[watchdog][execute] Waiting for interval=%s before monitoring group=%s service=%s again", service.Interval, service.Group, service.Name)
	}
	if !disableMonitoringLock {
		monitoringMutex.Unlock()
	}
}

// UpdateServiceStatuses updates the slice of service statuses
func UpdateServiceStatuses(service *core.Service, result *core.Result) {
	storage.Get().Insert(service, result)
}

// Shutdown stops monitoring all services
func Shutdown() {
	cancelFunc()
}