fix(watchdog): Close dangling file descriptors on shutdown and config reload (#544)

* fix(watchdog): Add functions to avoid dangling file descriptors

* Change function name and add comment under core/endpoint.go
- change the function name of CloseHTTPConnection() to Close()
- add some comments above Close() function

* Update core/endpoint.go

* Update core/endpoint.go

---------

Co-authored-by: Richard Cheng <richard_cheng@trendmicro.com>
Co-authored-by: TwiN <twin@linux.com>

core/endpoint.go

@@ -364,6 +364,15 @@ func (endpoint *Endpoint) call(result *Result) {
 	}
 }
 
+// Close HTTP connections between watchdog and endpoints to avoid dangling socket file descriptors
+// on configuration reload.
+// More context on https://github.com/TwiN/gatus/issues/536
+func (endpoint *Endpoint) Close() {
+	if endpoint.Type() == EndpointTypeHTTP {
+		client.GetHTTPClient(endpoint.ClientConfig).CloseIdleConnections()
+	}
+}
+
 func (endpoint *Endpoint) buildHTTPRequest() *http.Request {
 	var bodyBuffer *bytes.Buffer
 	if endpoint.GraphQL {

main.go

@@ -27,7 +27,7 @@ func main() {
 	go func() {
 		<-signalChannel
 		log.Println("Received termination signal, attempting to gracefully shut down")
-		stop()
+		stop(cfg)
 		save()
 		done <- true
 	}()
@@ -41,8 +41,8 @@ func start(cfg *config.Config) {
 	go listenToConfigurationFileChanges(cfg)
 }
 
-func stop() {
+func stop(cfg *config.Config) {
-	watchdog.Shutdown()
+	watchdog.Shutdown(cfg)
 	controller.Shutdown()
 }
 
@@ -89,7 +89,7 @@ func listenToConfigurationFileChanges(cfg *config.Config) {
 		time.Sleep(30 * time.Second)
 		if cfg.HasLoadedConfigurationBeenModified() {
 			log.Println("[main][listenToConfigurationFileChanges] Configuration file has been modified")
-			stop()
+			stop(cfg)
 			time.Sleep(time.Second) // Wait a bit to make sure everything is done.
 			save()
 			updatedConfig, err := loadConfiguration()
@@ -104,6 +104,7 @@ func listenToConfigurationFileChanges(cfg *config.Config) {
 					panic(err)
 				}
 			}
+			store.Get().Close()
 			initializeStorage(updatedConfig)
 			start(updatedConfig)
 			return

watchdog/watchdog.go

@@ -96,6 +96,10 @@ func UpdateEndpointStatuses(endpoint *core.Endpoint, result *core.Result) {
 }
 
 // Shutdown stops monitoring all endpoints
-func Shutdown() {
+func Shutdown(cfg *config.Config) {
+	// Disable all the old HTTP connections
+	for _, endpoint := range cfg.Endpoints {
+		endpoint.Close()
+	}
 	cancelFunc()
 }