From 8e1430276569c87dc1729d8c548221f89986fbda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henning=20Jan=C3=9Fen?= Date: Sat, 7 Jan 2023 05:46:19 +0100 Subject: [PATCH] feat: Support multiple configuration files (#389) * Allow configuration to be distributed * catch iteration errors when collecting config files * rm unused func * Fix suffix check for config loading * test configuration loading * GATUS_CONFIG_PATH can be a file or a directory now * Add deprecation note * Fix cs Co-authored-by: TwiN * cs fixes Co-authored-by: TwiN * cs fixes Co-authored-by: TwiN * cs fixes Co-authored-by: TwiN * cs + rm useless line Co-authored-by: TwiN * Update config/config.go Co-authored-by: TwiN --- README.md | 2 +- config/config.go | 134 ++++++++++++++++++++---------- config/config_test.go | 25 +++++- main.go | 14 ++-- test-conf/conf.d/endpoints.yaml | 24 ++++++ test-conf/conf.d/endpoints.yml | 29 +++++++ test-conf/config.yaml | 6 ++ test-conf/empty-conf.d/.gitignore | 0 8 files changed, 179 insertions(+), 55 deletions(-) create mode 100644 test-conf/conf.d/endpoints.yaml create mode 100644 test-conf/conf.d/endpoints.yml create mode 100644 test-conf/config.yaml create mode 100644 test-conf/empty-conf.d/.gitignore diff --git a/README.md b/README.md index 4b2ab8be..50bd48d5 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ If you want to create your own configuration, see [Docker](#docker) for informat By default, the configuration file is expected to be at `config/config.yaml`. -You can specify a custom path by setting the `GATUS_CONFIG_FILE` environment variable. +You can specify a custom path by setting the `GATUS_CONFIG_PATH` environment variable. If `GATUS_CONFIG_PATH` points to a directory, all `*.yaml` and `*.yml` files inside this directory and its subdirectories are concatenated. The previously used environment variable `GATUS_CONFIG_FILE` is deprecated but still works. Here's a simple example: ```yaml diff --git a/config/config.go b/config/config.go index 8fb4e9b3..35721721 100644 --- a/config/config.go +++ b/config/config.go @@ -3,8 +3,10 @@ package config import ( "errors" "fmt" + "io/fs" "log" "os" + "path/filepath" "time" "github.com/TwiN/gatus/v5/alerting" @@ -40,6 +42,9 @@ var ( // ErrInvalidSecurityConfig is an error returned when the security configuration is invalid ErrInvalidSecurityConfig = errors.New("invalid security configuration") + + // ErrEarlyReturn is returned to break out of a loop from a callback early + ErrEarlyReturn = errors.New("early escape") ) // Config is the main configuration structure @@ -84,7 +89,7 @@ type Config struct { // WARNING: This is in ALPHA and may change or be completely removed in the future Remote *remote.Config `yaml:"remote,omitempty"` - filePath string // path to the file from which config was loaded from + configPath string // path to the file or directory from which config was loaded lastFileModTime time.Time // last modification time } @@ -98,63 +103,80 @@ func (config *Config) GetEndpointByKey(key string) *core.Endpoint { return nil } -// HasLoadedConfigurationFileBeenModified returns whether the file that the +// HasLoadedConfigurationFileBeenModified returns whether one of the file that the // configuration has been loaded from has been modified since it was last read func (config Config) HasLoadedConfigurationFileBeenModified() bool { - if fileInfo, err := os.Stat(config.filePath); err == nil { - if !fileInfo.ModTime().IsZero() { - return config.lastFileModTime.Unix() != fileInfo.ModTime().Unix() - } + lastMod := config.lastFileModTime.Unix() + fileInfo, err := os.Stat(config.configPath) + if err != nil { + return false } - return false + if fileInfo.IsDir() { + err = walkConfigDir(config.configPath, func (path string, d fs.DirEntry, err error) error { + if info, err := d.Info(); err == nil && lastMod < info.ModTime().Unix() { + return ErrEarlyReturn + } + return nil + }) + return err == ErrEarlyReturn + } + return !fileInfo.ModTime().IsZero() && config.lastFileModTime.Unix() < fileInfo.ModTime().Unix() } // UpdateLastFileModTime refreshes Config.lastFileModTime func (config *Config) UpdateLastFileModTime() { - if fileInfo, err := os.Stat(config.filePath); err == nil { - if !fileInfo.ModTime().IsZero() { - config.lastFileModTime = fileInfo.ModTime() + config.lastFileModTime = time.Now() +} + +// LoadConfiguration loads the full configuration composed from the main configuration file +// and all composed configuration files +func LoadConfiguration(configPath string) (*Config, error) { + var composedContents []byte + var fileInfo os.FileInfo + var usedConfigPath string = "" + + for _, cpath := range []string{configPath, DefaultConfigurationFilePath, DefaultFallbackConfigurationFilePath} { + if len(cpath) == 0 { + continue } + + var err error + fileInfo, err = os.Stat(cpath) + if err != nil { + continue + } + + usedConfigPath = configPath + break + } + if len(usedConfigPath) == 0 { + return nil, ErrConfigFileNotFound + } + + if fileInfo.IsDir() { + walkConfigDir(configPath, func(path string, d fs.DirEntry, err error) error { + bytes, rerr := os.ReadFile(path) + if rerr == nil { + log.Printf("[config][Load] Reading configuration from configFile=%s", path) + composedContents = append(composedContents, bytes...) + } + return nil + }) } else { - log.Println("[config][UpdateLastFileModTime] Ran into error updating lastFileModTime:", err.Error()) - } -} - -// Load loads a custom configuration file -// Note that the misconfiguration of some fields may lead to panics. This is on purpose. -func Load(configFile string) (*Config, error) { - log.Printf("[config][Load] Reading configuration from configFile=%s", configFile) - cfg, err := readConfigurationFile(configFile) - if err != nil { - if os.IsNotExist(err) { - return nil, ErrConfigFileNotFound + bytes, serr := os.ReadFile(usedConfigPath) + if serr == nil { + log.Printf("[config][Load] Reading configuration from configFile=%s", configPath) + composedContents = bytes } - return nil, err } - cfg.filePath = configFile - cfg.UpdateLastFileModTime() - return cfg, nil -} -// LoadDefaultConfiguration loads the default configuration file -func LoadDefaultConfiguration() (*Config, error) { - cfg, err := Load(DefaultConfigurationFilePath) - if err != nil { - if err == ErrConfigFileNotFound { - return Load(DefaultFallbackConfigurationFilePath) - } - return nil, err + if len(composedContents) == 0 { + return nil, ErrConfigFileNotFound } - return cfg, nil -} - -func readConfigurationFile(fileName string) (config *Config, err error) { - var bytes []byte - if bytes, err = os.ReadFile(fileName); err == nil { - // file exists, so we'll parse it and return it - return parseAndValidateConfigBytes(bytes) - } - return + config, err := parseAndValidateConfigBytes(composedContents) + config.configPath = usedConfigPath + config.UpdateLastFileModTime() + return config, err } // parseAndValidateConfigBytes parses a Gatus configuration file into a Config struct and validates its parameters @@ -332,3 +354,25 @@ func validateAlertingConfig(alertingConfig *alerting.Config, endpoints []*core.E } log.Printf("[config][validateAlertingConfig] configuredProviders=%s; ignoredProviders=%s", validProviders, invalidProviders) } + + +// walkConfigDir is a wrapper for filepath.WalkDir that strips directories and non-config files +func walkConfigDir(path string, fn fs.WalkDirFunc) error { + if len(path) == 0 { + // If the user didn't provide a directory, we'll just use the default config file, so we can return nil now. + return nil + } + return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if d == nil || d.IsDir() { + return nil + } + ext := filepath.Ext(path) + if ext != ".yml" && ext != ".yaml" { + return nil + } + return fn(path, d, err) + }) +} diff --git a/config/config_test.go b/config/config_test.go index 8d8557d9..bd1f6ae7 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -29,19 +29,40 @@ import ( ) func TestLoadFileThatDoesNotExist(t *testing.T) { - _, err := Load("file-that-does-not-exist.yaml") + _, err := LoadConfiguration("file-that-does-not-exist.yaml") if err == nil { t.Error("Should've returned an error, because the file specified doesn't exist") } } func TestLoadDefaultConfigurationFile(t *testing.T) { - _, err := LoadDefaultConfiguration() + _, err := LoadConfiguration("") if err == nil { t.Error("Should've returned an error, because there's no configuration files at the default path nor the default fallback path") } } +func TestLoadConfigurationFile(t *testing.T) { + _, err := LoadConfiguration("../test-conf/config.yaml") + if nil != err { + t.Error("Should not have returned an error, because the configuration file exists at the provided path") + } +} + +func TestLoadDistributedConfiguration(t *testing.T) { + _, err := LoadConfiguration("../test-conf/conf.d/") + if nil != err { + t.Error("Should not have returned an error, because configuration files exist at the provided path for distributed files") + } +} + +func TestLoadCombinedConfiguration(t *testing.T) { + _, err := LoadConfiguration("../test-conf/empty-conf.d/") + if nil == err { + t.Error("Should have returned an error, because the configuration directory does not contain any configuration files") + } +} + func TestParseAndValidateConfigBytes(t *testing.T) { file := t.TempDir() + "/test.db" config, err := parseAndValidateConfigBytes([]byte(fmt.Sprintf(` diff --git a/main.go b/main.go index 36a5721d..5a4463d6 100644 --- a/main.go +++ b/main.go @@ -52,14 +52,14 @@ func save() { } } -func loadConfiguration() (cfg *config.Config, err error) { - customConfigFile := os.Getenv("GATUS_CONFIG_FILE") - if len(customConfigFile) > 0 { - cfg, err = config.Load(customConfigFile) - } else { - cfg, err = config.LoadDefaultConfiguration() +func loadConfiguration() (*config.Config, error) { + var configPath = os.Getenv("GATUS_CONFIG_PATH") + // Backwards compatibility + if len(configPath) == 0 { + configPath = os.Getenv("GATUS_CONFIG_FILE") } - return + + return config.LoadConfiguration(configPath) } // initializeStorage initializes the storage provider diff --git a/test-conf/conf.d/endpoints.yaml b/test-conf/conf.d/endpoints.yaml new file mode 100644 index 00000000..26cd685a --- /dev/null +++ b/test-conf/conf.d/endpoints.yaml @@ -0,0 +1,24 @@ +endpoints: + - name: front-end + group: core + url: "https://twin.sh/health" + interval: 5m + conditions: + - "[STATUS] == 200" + - "[BODY].status == UP" + - "[RESPONSE_TIME] < 150" + + - name: back-end + group: core + url: "https://example.org/" + interval: 5m + conditions: + - "[STATUS] == 200" + - "[CERTIFICATE_EXPIRATION] > 48h" + + - name: monitoring + group: internal + url: "https://example.org/" + interval: 5m + conditions: + - "[STATUS] == 200" diff --git a/test-conf/conf.d/endpoints.yml b/test-conf/conf.d/endpoints.yml new file mode 100644 index 00000000..e9519207 --- /dev/null +++ b/test-conf/conf.d/endpoints.yml @@ -0,0 +1,29 @@ +endpoints: + - name: nas + group: internal + url: "https://example.org/" + interval: 5m + conditions: + - "[STATUS] == 200" + + - name: example-dns-query + url: "8.8.8.8" # Address of the DNS server to use + interval: 5m + dns: + query-name: "example.com" + query-type: "A" + conditions: + - "[BODY] == 93.184.216.34" + - "[DNS_RCODE] == NOERROR" + + - name: icmp-ping + url: "icmp://example.org" + interval: 1m + conditions: + - "[CONNECTED] == true" + + - name: check-domain-expiration + url: "https://example.org/" + interval: 1h + conditions: + - "[DOMAIN_EXPIRATION] > 720h" diff --git a/test-conf/config.yaml b/test-conf/config.yaml new file mode 100644 index 00000000..0a428f94 --- /dev/null +++ b/test-conf/config.yaml @@ -0,0 +1,6 @@ +endpoints: + - name: example + url: https://example.org + interval: 30s + conditions: + - "[STATUS] == 200" diff --git a/test-conf/empty-conf.d/.gitignore b/test-conf/empty-conf.d/.gitignore new file mode 100644 index 00000000..e69de29b