Add support to envsub go management configurations (#2708)

This change allows users to reference environment variables using Go template format, like {{ .EnvName }}

Moved the previous file test code to file_suite_test.go.

management/cmd/management.go

@@ -475,7 +475,7 @@ func handlerFunc(gRPCHandler *grpc.Server, httpHandler http.Handler) http.Handle
 
 func loadMgmtConfig(ctx context.Context, mgmtConfigPath string) (*server.Config, error) {
 	loadedConfig := &server.Config{}
-	_, err := util.ReadJson(mgmtConfigPath, loadedConfig)
+	_, err := util.ReadJsonWithEnvSub(mgmtConfigPath, loadedConfig)
 	if err != nil {
 		return nil, err
 	}

util/file.go

@@ -1,11 +1,15 @@
 package util
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
+	"text/template"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -160,6 +164,55 @@ func ReadJson(file string, res interface{}) (interface{}, error) {
 	return res, nil
 }
 
+// ReadJsonWithEnvSub reads JSON config file and maps to a provided interface with environment variable substitution
+func ReadJsonWithEnvSub(file string, res interface{}) (interface{}, error) {
+	envVars := getEnvMap()
+
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	bs, err := io.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+
+	t, err := template.New("").Parse(string(bs))
+	if err != nil {
+		return nil, fmt.Errorf("error parsing template: %v", err)
+	}
+
+	var output bytes.Buffer
+	// Execute the template, substituting environment variables
+	err = t.Execute(&output, envVars)
+	if err != nil {
+		return nil, fmt.Errorf("error executing template: %v", err)
+	}
+
+	err = json.Unmarshal(output.Bytes(), &res)
+	if err != nil {
+		return nil, fmt.Errorf("failed parsing Json file after template was executed, err: %v", err)
+	}
+
+	return res, nil
+}
+
+// getEnvMap Convert the output of os.Environ() to a map
+func getEnvMap() map[string]string {
+	envMap := make(map[string]string)
+
+	for _, env := range os.Environ() {
+		parts := strings.SplitN(env, "=", 2)
+		if len(parts) == 2 {
+			envMap[parts[0]] = parts[1]
+		}
+	}
+
+	return envMap
+}
+
 // CopyFileContents copies contents of the given src file to the dst file
 func CopyFileContents(src, dst string) (err error) {
 	in, err := os.Open(src)

util/file_suite_test.go

@@ -0,0 +1,126 @@
+package util_test
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"io"
+	"os"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/netbirdio/netbird/util"
+)
+
+var _ = Describe("Client", func() {
+
+	var (
+		tmpDir string
+	)
+
+	type TestConfig struct {
+		SomeMap   map[string]string
+		SomeArray []string
+		SomeField int
+	}
+
+	BeforeEach(func() {
+		var err error
+		tmpDir, err = os.MkdirTemp("", "wiretrustee_util_test_tmp_*")
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		err := os.RemoveAll(tmpDir)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	Describe("Config", func() {
+		Context("in JSON format", func() {
+			It("should be written and read successfully", func() {
+
+				m := make(map[string]string)
+				m["key1"] = "value1"
+				m["key2"] = "value2"
+
+				arr := []string{"value1", "value2"}
+
+				written := &TestConfig{
+					SomeMap:   m,
+					SomeArray: arr,
+					SomeField: 99,
+				}
+
+				err := util.WriteJson(tmpDir+"/testconfig.json", written)
+				Expect(err).NotTo(HaveOccurred())
+
+				read, err := util.ReadJson(tmpDir+"/testconfig.json", &TestConfig{})
+				Expect(err).NotTo(HaveOccurred())
+				Expect(read).NotTo(BeNil())
+				Expect(read.(*TestConfig).SomeMap["key1"]).To(BeEquivalentTo(written.SomeMap["key1"]))
+				Expect(read.(*TestConfig).SomeMap["key2"]).To(BeEquivalentTo(written.SomeMap["key2"]))
+				Expect(read.(*TestConfig).SomeArray).To(ContainElements(arr))
+				Expect(read.(*TestConfig).SomeField).To(BeEquivalentTo(written.SomeField))
+
+			})
+		})
+	})
+
+	Describe("Copying file contents", func() {
+		Context("from one file to another", func() {
+			It("should be successful", func() {
+
+				src := tmpDir + "/copytest_src"
+				dst := tmpDir + "/copytest_dst"
+
+				err := util.WriteJson(src, []string{"1", "2", "3"})
+				Expect(err).NotTo(HaveOccurred())
+
+				err = util.CopyFileContents(src, dst)
+				Expect(err).NotTo(HaveOccurred())
+
+				hashSrc := md5.New()
+				hashDst := md5.New()
+
+				srcFile, err := os.Open(src)
+				Expect(err).NotTo(HaveOccurred())
+
+				dstFile, err := os.Open(dst)
+				Expect(err).NotTo(HaveOccurred())
+
+				_, err = io.Copy(hashSrc, srcFile)
+				Expect(err).NotTo(HaveOccurred())
+
+				_, err = io.Copy(hashDst, dstFile)
+				Expect(err).NotTo(HaveOccurred())
+
+				err = srcFile.Close()
+				Expect(err).NotTo(HaveOccurred())
+
+				err = dstFile.Close()
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(hex.EncodeToString(hashSrc.Sum(nil)[:16])).To(BeEquivalentTo(hex.EncodeToString(hashDst.Sum(nil)[:16])))
+			})
+		})
+	})
+
+	Describe("Handle config file without full path", func() {
+		Context("config file handling", func() {
+			It("should be successful", func() {
+				written := &TestConfig{
+					SomeField: 123,
+				}
+				cfgFile := "test_cfg.json"
+				defer os.Remove(cfgFile)
+
+				err := util.WriteJson(cfgFile, written)
+				Expect(err).NotTo(HaveOccurred())
+
+				read, err := util.ReadJson(cfgFile, &TestConfig{})
+				Expect(err).NotTo(HaveOccurred())
+				Expect(read).NotTo(BeNil())
+			})
+		})
+	})
+})

util/file_test.go

@@ -1,126 +1,198 @@
-package util_test
+package util
 
 import (
-	"crypto/md5"
-	"encoding/hex"
-	"io"
 	"os"
-
+	"reflect"
-	. "github.com/onsi/ginkgo"
+	"strings"
-	. "github.com/onsi/gomega"
+	"testing"
-
-	"github.com/netbirdio/netbird/util"
 )
 
-var _ = Describe("Client", func() {
+func TestReadJsonWithEnvSub(t *testing.T) {
-
+	type Config struct {
-	var (
+		CertFile     string `json:"CertFile"`
-		tmpDir string
+		Credentials  string `json:"Credentials"`
-	)
+		NestedOption struct {
-
+			URL string `json:"URL"`
-	type TestConfig struct {
+		} `json:"NestedOption"`
-		SomeMap   map[string]string
-		SomeArray []string
-		SomeField int
 	}
 
-	BeforeEach(func() {
+	type testCase struct {
-		var err error
+		name           string
-		tmpDir, err = os.MkdirTemp("", "wiretrustee_util_test_tmp_*")
+		envVars        map[string]string
-		Expect(err).NotTo(HaveOccurred())
+		jsonTemplate   string
-	})
+		expectedResult Config
+		expectError    bool
+		errorContains  string
+	}
 
-	AfterEach(func() {
+	tests := []testCase{
-		err := os.RemoveAll(tmpDir)
+		{
-		Expect(err).NotTo(HaveOccurred())
+			name: "All environment variables set",
-	})
+			envVars: map[string]string{
+				"CERT_FILE":   "/etc/certs/env_cert.crt",
+				"CREDENTIALS": "env_credentials",
+				"URL":         "https://env.testing.com",
+			},
+			jsonTemplate: `{
+			  "CertFile": "{{ .CERT_FILE }}",
+			  "Credentials": "{{ .CREDENTIALS }}",
+			  "NestedOption": {
+				   "URL": "{{ .URL }}"
+			  }
+			}`,
+			expectedResult: Config{
+				CertFile:    "/etc/certs/env_cert.crt",
+				Credentials: "env_credentials",
+				NestedOption: struct {
+					URL string `json:"URL"`
+				}{
+					URL: "https://env.testing.com",
+				},
+			},
+			expectError: false,
+		},
+		{
+			name: "Missing environment variable",
+			envVars: map[string]string{
+				"CERT_FILE":   "/etc/certs/env_cert.crt",
+				"CREDENTIALS": "env_credentials",
+				// "URL" is intentionally missing
+			},
+			jsonTemplate: `{
+			  "CertFile": "{{ .CERT_FILE }}",
+			  "Credentials": "{{ .CREDENTIALS }}",
+			  "NestedOption": {
+				   "URL": "{{ .URL }}"
+			  }
+			}`,
+			expectedResult: Config{
+				CertFile:    "/etc/certs/env_cert.crt",
+				Credentials: "env_credentials",
+				NestedOption: struct {
+					URL string `json:"URL"`
+				}{
+					URL: "<no value>",
+				},
+			},
+			expectError: false,
+		},
+		{
+			name: "Invalid JSON template",
+			envVars: map[string]string{
+				"CERT_FILE":   "/etc/certs/env_cert.crt",
+				"CREDENTIALS": "env_credentials",
+				"URL":         "https://env.testing.com",
+			},
+			jsonTemplate: `{
+			  "CertFile": "{{ .CERT_FILE }}",
+			  "Credentials": "{{ .CREDENTIALS }",
+			  "NestedOption": {
+				   "URL": "{{ .URL }}"
+			  }
+			}`, // Note the missing closing brace in "{{ .CREDENTIALS }"
+			expectedResult: Config{},
+			expectError:    true,
+			errorContains:  "unexpected \"}\" in operand",
+		},
+		{
+			name: "No substitutions",
+			envVars: map[string]string{
+				"CERT_FILE":   "/etc/certs/env_cert.crt",
+				"CREDENTIALS": "env_credentials",
+				"URL":         "https://env.testing.com",
+			},
+			jsonTemplate: `{
+			  "CertFile": "/etc/certs/cert.crt",
+			  "Credentials": "admnlknflkdasdf",
+			  "NestedOption" : {
+				   "URL": "https://testing.com"
+			  }
+			}`,
+			expectedResult: Config{
+				CertFile:    "/etc/certs/cert.crt",
+				Credentials: "admnlknflkdasdf",
+				NestedOption: struct {
+					URL string `json:"URL"`
+				}{
+					URL: "https://testing.com",
+				},
+			},
+			expectError: false,
+		},
+		{
+			name: "Should fail when Invalid characters in variables",
+			envVars: map[string]string{
+				"CERT_FILE":   `"/etc/certs/"cert".crt"`,
+				"CREDENTIALS": `env_credentia{ls}`,
+				"URL":         `https://env.testing.com?param={{value}}`,
+			},
+			jsonTemplate: `{
+			  "CertFile": "{{ .CERT_FILE }}",
+			  "Credentials": "{{ .CREDENTIALS }}",
+			  "NestedOption": {
+				   "URL": "{{ .URL }}"
+			  }
+			}`,
+			expectedResult: Config{
+				CertFile:    `"/etc/certs/"cert".crt"`,
+				Credentials: `env_credentia{ls}`,
+				NestedOption: struct {
+					URL string `json:"URL"`
+				}{
+					URL: `https://env.testing.com?param={{value}}`,
+				},
+			},
+			expectError: true,
+		},
+	}
 
-	Describe("Config", func() {
+	for _, tc := range tests {
-		Context("in JSON format", func() {
+		tc := tc
-			It("should be written and read successfully", func() {
+		t.Run(tc.name, func(t *testing.T) {
+			for key, value := range tc.envVars {
+				t.Setenv(key, value)
+			}
 
-				m := make(map[string]string)
+			tempFile, err := os.CreateTemp("", "config*.json")
-				m["key1"] = "value1"
+			if err != nil {
-				m["key2"] = "value2"
+				t.Fatalf("Failed to create temp file: %v", err)
+			}
 
-				arr := []string{"value1", "value2"}
+			defer func() {
-
+				err = os.Remove(tempFile.Name())
-				written := &TestConfig{
+				if err != nil {
-					SomeMap:   m,
+					t.Logf("Failed to remove temp file: %v", err)
-					SomeArray: arr,
-					SomeField: 99,
 				}
+			}()
 
-				err := util.WriteJson(tmpDir+"/testconfig.json", written)
+			_, err = tempFile.WriteString(tc.jsonTemplate)
-				Expect(err).NotTo(HaveOccurred())
+			if err != nil {
+				t.Fatalf("Failed to write to temp file: %v", err)
+			}
+			err = tempFile.Close()
+			if err != nil {
+				t.Fatalf("Failed to close temp file: %v", err)
+			}
 
-				read, err := util.ReadJson(tmpDir+"/testconfig.json", &TestConfig{})
+			var result Config
-				Expect(err).NotTo(HaveOccurred())
-				Expect(read).NotTo(BeNil())
-				Expect(read.(*TestConfig).SomeMap["key1"]).To(BeEquivalentTo(written.SomeMap["key1"]))
-				Expect(read.(*TestConfig).SomeMap["key2"]).To(BeEquivalentTo(written.SomeMap["key2"]))
-				Expect(read.(*TestConfig).SomeArray).To(ContainElements(arr))
-				Expect(read.(*TestConfig).SomeField).To(BeEquivalentTo(written.SomeField))
 
-			})
+			_, err = ReadJsonWithEnvSub(tempFile.Name(), &result)
-		})
-	})
 
-	Describe("Copying file contents", func() {
+			if tc.expectError {
-		Context("from one file to another", func() {
+				if err == nil {
-			It("should be successful", func() {
+					t.Fatalf("Expected error but got none")
-
-				src := tmpDir + "/copytest_src"
-				dst := tmpDir + "/copytest_dst"
-
-				err := util.WriteJson(src, []string{"1", "2", "3"})
-				Expect(err).NotTo(HaveOccurred())
-
-				err = util.CopyFileContents(src, dst)
-				Expect(err).NotTo(HaveOccurred())
-
-				hashSrc := md5.New()
-				hashDst := md5.New()
-
-				srcFile, err := os.Open(src)
-				Expect(err).NotTo(HaveOccurred())
-
-				dstFile, err := os.Open(dst)
-				Expect(err).NotTo(HaveOccurred())
-
-				_, err = io.Copy(hashSrc, srcFile)
-				Expect(err).NotTo(HaveOccurred())
-
-				_, err = io.Copy(hashDst, dstFile)
-				Expect(err).NotTo(HaveOccurred())
-
-				err = srcFile.Close()
-				Expect(err).NotTo(HaveOccurred())
-
-				err = dstFile.Close()
-				Expect(err).NotTo(HaveOccurred())
-
-				Expect(hex.EncodeToString(hashSrc.Sum(nil)[:16])).To(BeEquivalentTo(hex.EncodeToString(hashDst.Sum(nil)[:16])))
-			})
-		})
-	})
-
-	Describe("Handle config file without full path", func() {
-		Context("config file handling", func() {
-			It("should be successful", func() {
-				written := &TestConfig{
-					SomeField: 123,
 				}
-				cfgFile := "test_cfg.json"
+				if !strings.Contains(err.Error(), tc.errorContains) {
-				defer os.Remove(cfgFile)
+					t.Errorf("Expected error containing '%s', but got '%v'", tc.errorContains, err)
-
+				}
-				err := util.WriteJson(cfgFile, written)
+			} else {
-				Expect(err).NotTo(HaveOccurred())
+				if err != nil {
-
+					t.Fatalf("ReadJsonWithEnvSub failed: %v", err)
-				read, err := util.ReadJson(cfgFile, &TestConfig{})
+				}
-				Expect(err).NotTo(HaveOccurred())
+				if !reflect.DeepEqual(result, tc.expectedResult) {
-				Expect(read).NotTo(BeNil())
+					t.Errorf("Result does not match expected.\nGot: %+v\nExpected: %+v", result, tc.expectedResult)
-			})
+				}
+			}
 		})
-	})
+	}
-})
+}