feat: Support multiple configuration files (#389)

* Allow configuration to be distributed

* catch iteration errors when collecting config files

* rm unused func

* Fix suffix check for config loading

* test configuration loading

* GATUS_CONFIG_PATH can be a file or a directory now

* Add deprecation note

* Fix cs

Co-authored-by: TwiN <twin@linux.com>

* cs fixes

Co-authored-by: TwiN <twin@linux.com>

* cs fixes

Co-authored-by: TwiN <twin@linux.com>

* cs fixes

Co-authored-by: TwiN <twin@linux.com>

* cs + rm useless line

Co-authored-by: TwiN <twin@linux.com>

* Update config/config.go

Co-authored-by: TwiN <twin@linux.com>

README.md

@@ -149,7 +149,7 @@ If you want to create your own configuration, see [Docker](#docker) for informat
 
 By default, the configuration file is expected to be at `config/config.yaml`.
 
-You can specify a custom path by setting the `GATUS_CONFIG_FILE` environment variable.
+You can specify a custom path by setting the `GATUS_CONFIG_PATH` environment variable. If `GATUS_CONFIG_PATH` points to a directory, all `*.yaml` and `*.yml` files inside this directory and its subdirectories are concatenated. The previously used environment variable `GATUS_CONFIG_FILE` is deprecated but still works.
 
 Here's a simple example:
 ```yaml

config/config.go

@@ -3,8 +3,10 @@ package config
 import (
 	"errors"
 	"fmt"
+	"io/fs"
 	"log"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/TwiN/gatus/v5/alerting"
@@ -40,6 +42,9 @@ var (
 
 	// ErrInvalidSecurityConfig is an error returned when the security configuration is invalid
 	ErrInvalidSecurityConfig = errors.New("invalid security configuration")
+
+	// ErrEarlyReturn is returned to break out of a loop from a callback early
+	ErrEarlyReturn = errors.New("early escape")
 )
 
 // Config is the main configuration structure
@@ -84,7 +89,7 @@ type Config struct {
 	// WARNING: This is in ALPHA and may change or be completely removed in the future
 	Remote *remote.Config `yaml:"remote,omitempty"`
 
-	filePath        string    // path to the file from which config was loaded from
+	configPath      string    // path to the file or directory from which config was loaded
 	lastFileModTime time.Time // last modification time
 }
 
@@ -98,63 +103,80 @@ func (config *Config) GetEndpointByKey(key string) *core.Endpoint {
 	return nil
 }
 
-// HasLoadedConfigurationFileBeenModified returns whether the file that the
+// HasLoadedConfigurationFileBeenModified returns whether one of the file that the
 // configuration has been loaded from has been modified since it was last read
 func (config Config) HasLoadedConfigurationFileBeenModified() bool {
-	if fileInfo, err := os.Stat(config.filePath); err == nil {
-		if !fileInfo.ModTime().IsZero() {
-			return config.lastFileModTime.Unix() != fileInfo.ModTime().Unix()
-		}
+	lastMod := config.lastFileModTime.Unix()
+	fileInfo, err := os.Stat(config.configPath)
+	if err != nil {
+		return false
 	}
-	return false
+	if fileInfo.IsDir() {
+		err = walkConfigDir(config.configPath, func (path string, d fs.DirEntry, err error) error {
+			if info, err := d.Info(); err == nil && lastMod < info.ModTime().Unix() {
+				return ErrEarlyReturn
+			}
+			return nil
+		})
+		return err == ErrEarlyReturn
+	}
+	return !fileInfo.ModTime().IsZero() && config.lastFileModTime.Unix() < fileInfo.ModTime().Unix()
 }
 
 // UpdateLastFileModTime refreshes Config.lastFileModTime
 func (config *Config) UpdateLastFileModTime() {
-	if fileInfo, err := os.Stat(config.filePath); err == nil {
-		if !fileInfo.ModTime().IsZero() {
-			config.lastFileModTime = fileInfo.ModTime()
+	config.lastFileModTime = time.Now()
+}
+
+// LoadConfiguration loads the full configuration composed from the main configuration file
+// and all composed configuration files
+func LoadConfiguration(configPath string) (*Config, error) {
+	var composedContents []byte
+	var fileInfo os.FileInfo
+	var usedConfigPath string = ""
+
+	for _, cpath := range []string{configPath, DefaultConfigurationFilePath, DefaultFallbackConfigurationFilePath} {
+		if len(cpath) == 0 {
+			continue
 		}
+
+		var err error
+		fileInfo, err = os.Stat(cpath)
+		if err != nil {
+			continue
+		}
+
+		usedConfigPath = configPath
+		break
+	}
+	if len(usedConfigPath) == 0 {
+		return nil, ErrConfigFileNotFound
+	}
+
+	if fileInfo.IsDir() {
+		walkConfigDir(configPath, func(path string, d fs.DirEntry, err error) error {
+			bytes, rerr := os.ReadFile(path)
+			if rerr == nil {
+				log.Printf("[config][Load] Reading configuration from configFile=%s", path)
+				composedContents = append(composedContents, bytes...)
+			}
+			return nil
+		})
 	} else {
-		log.Println("[config][UpdateLastFileModTime] Ran into error updating lastFileModTime:", err.Error())
-	}
-}
-
-// Load loads a custom configuration file
-// Note that the misconfiguration of some fields may lead to panics. This is on purpose.
-func Load(configFile string) (*Config, error) {
-	log.Printf("[config][Load] Reading configuration from configFile=%s", configFile)
-	cfg, err := readConfigurationFile(configFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, ErrConfigFileNotFound
+		bytes, serr := os.ReadFile(usedConfigPath)
+		if serr == nil {
+			log.Printf("[config][Load] Reading configuration from configFile=%s", configPath)
+			composedContents = bytes
 		}
-		return nil, err
 	}
-	cfg.filePath = configFile
-	cfg.UpdateLastFileModTime()
-	return cfg, nil
-}
 
-// LoadDefaultConfiguration loads the default configuration file
-func LoadDefaultConfiguration() (*Config, error) {
-	cfg, err := Load(DefaultConfigurationFilePath)
-	if err != nil {
-		if err == ErrConfigFileNotFound {
-			return Load(DefaultFallbackConfigurationFilePath)
-		}
-		return nil, err
+	if len(composedContents) == 0 {
+		return nil, ErrConfigFileNotFound
 	}
-	return cfg, nil
-}
-
-func readConfigurationFile(fileName string) (config *Config, err error) {
-	var bytes []byte
-	if bytes, err = os.ReadFile(fileName); err == nil {
-		// file exists, so we'll parse it and return it
-		return parseAndValidateConfigBytes(bytes)
-	}
-	return
+	config, err := parseAndValidateConfigBytes(composedContents)
+	config.configPath = usedConfigPath
+	config.UpdateLastFileModTime()
+	return config, err
 }
 
 // parseAndValidateConfigBytes parses a Gatus configuration file into a Config struct and validates its parameters
@@ -332,3 +354,25 @@ func validateAlertingConfig(alertingConfig *alerting.Config, endpoints []*core.E
 	}
 	log.Printf("[config][validateAlertingConfig] configuredProviders=%s; ignoredProviders=%s", validProviders, invalidProviders)
 }
+
+
+// walkConfigDir is a wrapper for filepath.WalkDir that strips directories and non-config files
+func walkConfigDir(path string, fn fs.WalkDirFunc) error {
+	if len(path) == 0 {
+		// If the user didn't provide a directory, we'll just use the default config file, so we can return nil now.
+		return nil
+	}
+	return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return nil
+		}
+		if d == nil || d.IsDir() {
+			return nil
+		}
+		ext := filepath.Ext(path)
+		if ext != ".yml" && ext != ".yaml" {
+			return nil
+		}
+		return fn(path, d, err)
+	})
+}

config/config_test.go

@@ -29,19 +29,40 @@ import (
 )
 
 func TestLoadFileThatDoesNotExist(t *testing.T) {
-	_, err := Load("file-that-does-not-exist.yaml")
+	_, err := LoadConfiguration("file-that-does-not-exist.yaml")
 	if err == nil {
 		t.Error("Should've returned an error, because the file specified doesn't exist")
 	}
 }
 
 func TestLoadDefaultConfigurationFile(t *testing.T) {
-	_, err := LoadDefaultConfiguration()
+	_, err := LoadConfiguration("")
 	if err == nil {
 		t.Error("Should've returned an error, because there's no configuration files at the default path nor the default fallback path")
 	}
 }
 
+func TestLoadConfigurationFile(t *testing.T) {
+	_, err := LoadConfiguration("../test-conf/config.yaml")
+	if nil != err {
+		t.Error("Should not have returned an error, because the configuration file exists at the provided path")
+	}
+}
+
+func TestLoadDistributedConfiguration(t *testing.T) {
+	_, err := LoadConfiguration("../test-conf/conf.d/")
+	if nil != err {
+		t.Error("Should not have returned an error, because configuration files exist at the provided path for distributed files")
+	}
+}
+
+func TestLoadCombinedConfiguration(t *testing.T) {
+	_, err := LoadConfiguration("../test-conf/empty-conf.d/")
+	if nil == err {
+		t.Error("Should have returned an error, because the configuration directory does not contain any configuration files")
+	}
+}
+
 func TestParseAndValidateConfigBytes(t *testing.T) {
 	file := t.TempDir() + "/test.db"
 	config, err := parseAndValidateConfigBytes([]byte(fmt.Sprintf(`

main.go

@@ -52,14 +52,14 @@ func save() {
 	}
 }
 
-func loadConfiguration() (cfg *config.Config, err error) {
-	customConfigFile := os.Getenv("GATUS_CONFIG_FILE")
-	if len(customConfigFile) > 0 {
-		cfg, err = config.Load(customConfigFile)
-	} else {
-		cfg, err = config.LoadDefaultConfiguration()
+func loadConfiguration() (*config.Config, error) {
+	var configPath = os.Getenv("GATUS_CONFIG_PATH")
+	// Backwards compatibility
+	if len(configPath) == 0 {
+		configPath = os.Getenv("GATUS_CONFIG_FILE")
 	}
-	return
+
+	return config.LoadConfiguration(configPath)
 }
 
 // initializeStorage initializes the storage provider

test-conf/conf.d/endpoints.yaml

@@ -0,0 +1,24 @@
+endpoints:
+  - name: front-end
+    group: core
+    url: "https://twin.sh/health"
+    interval: 5m
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].status == UP"
+      - "[RESPONSE_TIME] < 150"
+
+  - name: back-end
+    group: core
+    url: "https://example.org/"
+    interval: 5m
+    conditions:
+      - "[STATUS] == 200"
+      - "[CERTIFICATE_EXPIRATION] > 48h"
+
+  - name: monitoring
+    group: internal
+    url: "https://example.org/"
+    interval: 5m
+    conditions:
+      - "[STATUS] == 200"

test-conf/conf.d/endpoints.yml

@@ -0,0 +1,29 @@
+endpoints:
+  - name: nas
+    group: internal
+    url: "https://example.org/"
+    interval: 5m
+    conditions:
+      - "[STATUS] == 200"
+
+  - name: example-dns-query
+    url: "8.8.8.8" # Address of the DNS server to use
+    interval: 5m
+    dns:
+      query-name: "example.com"
+      query-type: "A"
+    conditions:
+      - "[BODY] == 93.184.216.34"
+      - "[DNS_RCODE] == NOERROR"
+
+  - name: icmp-ping
+    url: "icmp://example.org"
+    interval: 1m
+    conditions:
+      - "[CONNECTED] == true"
+
+  - name: check-domain-expiration
+    url: "https://example.org/"
+    interval: 1h
+    conditions:
+      - "[DOMAIN_EXPIRATION] > 720h"

test-conf/config.yaml

@@ -0,0 +1,6 @@
+endpoints:
+  - name: example
+    url: https://example.org
+    interval: 30s
+    conditions:
+      - "[STATUS] == 200"