fix(watchdog): Close dangling file descriptors on shutdown and config reload (#544)

* fix(watchdog): Add functions to avoid dangling file descriptors

* Change function name and add comment under core/endpoint.go
- change the function name of CloseHTTPConnection() to Close()
- add some comments above Close() function

* Update core/endpoint.go

* Update core/endpoint.go

---------

Co-authored-by: Richard Cheng <richard_cheng@trendmicro.com>
Co-authored-by: TwiN <twin@linux.com>
This commit is contained in:
I-HSIN Cheng 2023-08-05 06:30:15 +08:00 committed by GitHub
parent 640e455d33
commit 34313bec7e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 19 additions and 5 deletions

View File

@ -364,6 +364,15 @@ func (endpoint *Endpoint) call(result *Result) {
}
}
// Close HTTP connections between watchdog and endpoints to avoid dangling socket file descriptors
// on configuration reload.
// More context on https://github.com/TwiN/gatus/issues/536
func (endpoint *Endpoint) Close() {
if endpoint.Type() == EndpointTypeHTTP {
client.GetHTTPClient(endpoint.ClientConfig).CloseIdleConnections()
}
}
func (endpoint *Endpoint) buildHTTPRequest() *http.Request {
var bodyBuffer *bytes.Buffer
if endpoint.GraphQL {

View File

@ -27,7 +27,7 @@ func main() {
go func() {
<-signalChannel
log.Println("Received termination signal, attempting to gracefully shut down")
stop()
stop(cfg)
save()
done <- true
}()
@ -41,8 +41,8 @@ func start(cfg *config.Config) {
go listenToConfigurationFileChanges(cfg)
}
func stop() {
watchdog.Shutdown()
func stop(cfg *config.Config) {
watchdog.Shutdown(cfg)
controller.Shutdown()
}
@ -89,7 +89,7 @@ func listenToConfigurationFileChanges(cfg *config.Config) {
time.Sleep(30 * time.Second)
if cfg.HasLoadedConfigurationBeenModified() {
log.Println("[main][listenToConfigurationFileChanges] Configuration file has been modified")
stop()
stop(cfg)
time.Sleep(time.Second) // Wait a bit to make sure everything is done.
save()
updatedConfig, err := loadConfiguration()
@ -104,6 +104,7 @@ func listenToConfigurationFileChanges(cfg *config.Config) {
panic(err)
}
}
store.Get().Close()
initializeStorage(updatedConfig)
start(updatedConfig)
return

View File

@ -96,6 +96,10 @@ func UpdateEndpointStatuses(endpoint *core.Endpoint, result *core.Result) {
}
// Shutdown stops monitoring all endpoints
func Shutdown() {
func Shutdown(cfg *config.Config) {
// Disable all the old HTTP connections
for _, endpoint := range cfg.Endpoints {
endpoint.Close()
}
cancelFunc()
}