From c92c141b516f9c65f71d28ae9dfdb255ed779f59 Mon Sep 17 00:00:00 2001
From: Zeyad Tamimi <zeyadtamimi@outlook.com>
Date: Sat, 22 Feb 2025 11:51:05 -0800
Subject: [PATCH] Added include configuration support

A relatively straight forward change that adds "include" key support in
the main configuration file.

The config parser uses the list under this key to open the included
configuration files parse them as configs and append their jobs to the
main config file.
---
 internal/config/config.go                     | 71 ++++++++++++++++++-
 internal/config/config_include_test.go        | 43 +++++++++++
 internal/config/samples/include.d/snap.yml    | 14 ++++
 internal/config/samples/include.yml           |  2 +
 internal/config/samples/include_directory.yml |  2 +
 5 files changed, 131 insertions(+), 1 deletion(-)
 create mode 100644 internal/config/config_include_test.go
 create mode 100644 internal/config/samples/include.d/snap.yml
 create mode 100644 internal/config/samples/include.yml
 create mode 100644 internal/config/samples/include_directory.yml

diff --git a/internal/config/config.go b/internal/config/config.go
index 7722c6a..b9138b9 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -5,6 +5,8 @@ import (
 	"log/syslog"
 	"os"
 	"time"
+	"path/filepath"
+	pathpkg "path"
 
 	"github.com/pkg/errors"
 	"github.com/robfig/cron/v3"
@@ -24,6 +26,7 @@ const (
 type Config struct {
 	Jobs   []JobEnum `yaml:"jobs,optional"`
 	Global *Global   `yaml:"global,optional,fromdefaults"`
+	Include   []string `yaml:"include,optional"`
 }
 
 func (c *Config) Job(name string) (*JobEnum, error) {
@@ -657,6 +660,7 @@ var ConfigFileDefaultLocations = []string{
 
 func ParseConfig(path string) (i *Config, err error) {
 
+	// Parse main configuration file
 	if path == "" {
 		// Try default locations
 		for _, l := range ConfigFileDefaultLocations {
@@ -679,7 +683,71 @@ func ParseConfig(path string) (i *Config, err error) {
 		return
 	}
 
-	return ParseConfigBytes(bytes)
+	i, err = ParseConfigBytes(bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	err = ExpandConfigInclude(path, i)
+	if err != nil {
+		return nil, err
+	}
+
+	return i, err
+}
+
+func ExpandConfigInclude(configPath string, config *Config) (err error) {
+	if config == nil {
+		return nil
+	}
+
+	var includeConfigPaths []string
+	for  _, path := range config.Include {
+		if configPath[0] != '/' {
+			path = pathpkg.Join(pathpkg.Dir(configPath), path)
+		}
+
+		stat, statErr := os.Stat(path)
+		if statErr != nil {
+			return errors.Errorf("Could not open included configuration path: %s", path)
+		}
+
+		if stat.Mode().IsDir() {
+			directoryPaths, err := filepath.Glob(path + "/*.yml")
+			if err != nil {
+				return err
+			}
+
+			includeConfigPaths = append(includeConfigPaths, directoryPaths...)
+		} else if stat.Mode().IsRegular() {
+			if extention := filepath.Ext(path); extention != ".yml" {
+				return errors.Errorf("Only .yml files can be included: %s", path)
+			}
+			includeConfigPaths = append(includeConfigPaths, path)
+		} else {
+			return errors.Errorf("Only directories or .yml files can be included: %s", path)
+		}
+	}
+
+	for _, path := range includeConfigPaths {
+		var bytes []byte
+		if bytes, err = os.ReadFile(path); err != nil {
+			return err
+		}
+
+		includedConfig, err := ParseConfigBytes(bytes)
+		if err != nil {
+			return err
+		}
+
+		if len(includedConfig.Include) > 0 {
+			return  errors.Errorf("Included configuration files cannot include other files: %s", path)
+		}
+
+		config.Jobs = append(config.Jobs, includedConfig.Jobs...)
+	}
+
+	return nil
 }
 
 func ParseConfigBytes(bytes []byte) (*Config, error) {
@@ -687,6 +755,7 @@ func ParseConfigBytes(bytes []byte) (*Config, error) {
 	if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
 		return nil, err
 	}
+
 	if c != nil {
 		return c, nil
 	}
diff --git a/internal/config/config_include_test.go b/internal/config/config_include_test.go
new file mode 100644
index 0000000..cd2e5c0
--- /dev/null
+++ b/internal/config/config_include_test.go
@@ -0,0 +1,43 @@
+package config
+
+import (
+	"testing"
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIncludeSingle(t *testing.T) {
+	path := "./samples/include.yml"
+
+	t.Run(path, func(t *testing.T) {
+		config, err := ParseConfig(path)
+		if err != nil {
+			t.Errorf("error parsing %s:\n%+v", path, err)
+		}
+	
+		require.NotNil(t, config)
+		require.NotNil(t, config.Global)
+		require.NotEmpty(t, config.Jobs)
+
+		t.Logf("file: %s", path)
+		t.Log(pretty.Sprint(config))
+	})
+}
+
+func TestIncludeDirectory(t *testing.T) {
+	path := "./samples/include_directory.yml"
+
+	t.Run(path, func(t *testing.T) {
+		config, err := ParseConfig(path)
+		if err != nil {
+			t.Errorf("error parsing %s:\n%+v", path, err)
+		}
+	
+		require.NotNil(t, config)
+		require.NotNil(t, config.Global)
+		require.NotEmpty(t, config.Jobs)
+
+		t.Logf("file: %s", path)
+		t.Log(pretty.Sprint(config))
+	})
+}
diff --git a/internal/config/samples/include.d/snap.yml b/internal/config/samples/include.d/snap.yml
new file mode 100644
index 0000000..6775b01
--- /dev/null
+++ b/internal/config/samples/include.d/snap.yml
@@ -0,0 +1,14 @@
+jobs:
+- name: snapjob
+  type: snap
+  filesystems: {
+    "tank/frequently_changed<": true,
+  }
+  snapshotting:
+    type: periodic
+    interval: 2m
+    prefix: zrepl_snapjob_
+  pruning:
+    keep:
+      - type: last_n
+        count: 60
diff --git a/internal/config/samples/include.yml b/internal/config/samples/include.yml
new file mode 100644
index 0000000..c281c52
--- /dev/null
+++ b/internal/config/samples/include.yml
@@ -0,0 +1,2 @@
+include:
+  - ./snap.yml
diff --git a/internal/config/samples/include_directory.yml b/internal/config/samples/include_directory.yml
new file mode 100644
index 0000000..a6795f9
--- /dev/null
+++ b/internal/config/samples/include_directory.yml
@@ -0,0 +1,2 @@
+include:
+  - ./include.d