WIP new config format

Gopkg.lock

@@ -227,6 +227,14 @@
   revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0"
   version = "v1.1.4"
 
+[[projects]]
+  branch = "v2"
+  digest = "1:f0620375dd1f6251d9973b5f2596228cc8042e887cd7f827e4220bc1ce8c30e2"
+  name = "github.com/zrepl/yaml-config"
+  packages = ["."]
+  pruneopts = ""
+  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
+
 [[projects]]
   branch = "master"
   digest = "1:9c286cf11d0ca56368185bada5dd6d97b6be4648fc26c354fcba8df7293718f7"
@@ -256,6 +264,7 @@
     "github.com/spf13/cobra",
     "github.com/stretchr/testify/assert",
     "github.com/stretchr/testify/require",
+    "github.com/zrepl/yaml-config",
   ]
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -38,7 +38,7 @@ ignored = [ "github.com/inconshreveable/mousetrap" ]
 
 [[constraint]]
   branch = "v2"
-  name = "github.com/go-yaml/yaml"
+  name = "github.com/zrepl/yaml-config"
 
 [[constraint]]
   name = "github.com/go-logfmt/logfmt"

cmd/config/config.go

@@ -0,0 +1,223 @@
+package config
+
+import (
+	"github.com/zrepl/yaml-config"
+	"fmt"
+	"time"
+	"os"
+	"github.com/pkg/errors"
+	"io/ioutil"
+)
+
+type NodeEnum struct {
+	Ret interface{}
+}
+
+type PushNode struct {
+	Type string `yaml:"type"`
+	Replication PushReplication `yaml:"replication"`
+	Snapshotting Snapshotting `yaml:"snapshotting"`
+	Pruning Pruning `yaml:"pruning"`
+	Global Global `yaml:"global"`
+}
+
+type SinkNode struct {
+	Type string `yaml:"type"`
+	Replication SinkReplication `yaml:"replication"`
+	Global Global `yaml:"global"`
+}
+
+type PushReplication struct {
+	Connect ConnectEnum `yaml:"connect"`
+	Filesystems map[string]bool `yaml:"filesystems"`
+}
+
+type SinkReplication struct {
+	RootDataset string `yaml:"root_dataset"`
+	Serve ServeEnum `yaml:"serve"`
+}
+
+type Snapshotting struct {
+	SnapshotPrefix string `yaml:"snapshot_prefix"`
+	Interval time.Duration `yaml:"interval"`
+}
+
+type Pruning struct {
+	KeepLocal []PruningEnum `yaml:"keep_local"`
+	KeepRemote []PruningEnum `yaml:"keep_remote"`
+}
+
+type Global struct {
+	Logging []LoggingOutlet `yaml:"logging"`
+}
+
+type ConnectEnum struct {
+	Ret interface{}
+}
+
+type TCPConnect struct {
+	Type string `yaml:"type"`
+	Address string `yaml:"address"`
+}
+
+type TLSConnect struct {
+	Type string `yaml:"type"`
+	Address string `yaml:"address"`
+	Ca string `yaml:"ca"`
+	Cert string `yaml:"cert"`
+	Key string `yaml:"key"`
+}
+
+type ServeEnum struct {
+	Ret interface{}
+}
+
+type TCPServe struct {
+	Type string `yaml:"type"`
+	Listen string `yaml:"listen"`
+	Clients map[string]string `yaml:"clients"`
+}
+
+type TLSServe struct {
+	Type string `yaml:"type"`
+	Listen string `yaml:"listen"`
+	Ca string `yaml:"ca"`
+	Cert string `yaml:"cert"`
+	Key string `yaml:"key"`
+}
+
+type PruningEnum struct {
+	Ret interface{}
+}
+
+type PruneKeepNotReplicated struct {
+	Type string `yaml:"type"`
+}
+
+type PruneKeepLastN struct {
+	Type string `yaml:"type"`
+	Count int `yaml:"count"`
+}
+
+type PruneGrid struct {
+	Type string `yaml:"type"`
+	Grid string `yaml:"grid"`
+}
+
+type LoggingOutlet struct {
+	Outlet LoggingOutletEnum `yaml:"outlet"`
+	Level string `yaml:"level"`
+	Format string `yaml:"format"`
+}
+
+type LoggingOutletEnum struct {
+	Ret interface{}
+}
+
+type StdoutLoggingOutlet struct {
+	Type string `yaml:"type"`
+	Time bool `yaml:"time"`
+}
+
+type SyslogLoggingOutlet struct {
+	Type string `yaml:"type"`
+	RetryInterval time.Duration `yaml:"retry_interval"`
+}
+
+func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
+	var in struct {
+		Type string
+	}
+	if err := u(&in, true); err != nil {
+		return nil, err
+	}
+	if in.Type == "" {
+		return nil, &yaml.TypeError{[]string{"must specify type"}}
+	}
+
+	v, ok := types[in.Type]
+	if !ok {
+		return nil, &yaml.TypeError{[]string{fmt.Sprintf("invalid type name %q", in.Type)}}
+	}
+	if err := u(v, false); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+func (t *NodeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"push": &PushNode{},
+		"sink": &SinkNode{},
+	})
+	return
+}
+
+func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp": &TCPConnect{},
+		"tls": &TLSConnect{},
+	})
+	return
+}
+
+func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp": &TCPServe{},
+		"tls": &TLSServe{},
+	})
+	return
+}
+
+func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"not_replicated": &PruneKeepNotReplicated{},
+		"last_n": &PruneKeepLastN{},
+		"grid": &PruneGrid{},
+	})
+	return
+}
+
+func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"stdout": &StdoutLoggingOutlet{},
+		"syslog": &SyslogLoggingOutlet{},
+	})
+	return
+}
+
+var ConfigFileDefaultLocations = []string{
+	"/etc/zrepl/zrepl.yml",
+	"/usr/local/etc/zrepl/zrepl.yml",
+}
+
+func ParseConfig(path string) (i NodeEnum, err error) {
+
+	if path == "" {
+		// Try default locations
+		for _, l := range ConfigFileDefaultLocations {
+			stat, statErr := os.Stat(l)
+			if statErr != nil {
+				continue
+			}
+			if !stat.Mode().IsRegular() {
+				err = errors.Errorf("file at default location is not a regular file: %s", l)
+				return
+			}
+			path = l
+			break
+		}
+	}
+
+	var bytes []byte
+
+	if bytes, err = ioutil.ReadFile(path); err != nil {
+		return
+	}
+
+	if err = yaml.UnmarshalStrict(bytes, &i); err != nil {
+		return
+	}
+
+	return
+}

cmd/config/config_test.go

@@ -0,0 +1,27 @@
+package config
+
+import (
+	"testing"
+	"github.com/kr/pretty"
+	"path/filepath"
+)
+
+func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
+	paths, err := filepath.Glob("./samples/*")
+	if err != nil {
+		t.Errorf("glob failed: %+v", err)
+	}
+
+	for _, p := range paths {
+
+		c, err := ParseConfig(p)
+		if err != nil {
+			t.Errorf("error parsing %s:\n%+v", p, err)
+		}
+
+		t.Logf("file: %s", p)
+		t.Log(pretty.Sprint(c))
+
+	}
+
+}

cmd/config/samples/push.yml

@@ -0,0 +1,30 @@
+type: push
+replication:
+  connect:
+    type: tcp
+    address: "backup-server.foo.bar:8888"
+  filesystems: {
+    "<": true,
+    "tmp": false
+  }
+snapshotting:
+  snapshot_prefix: zrepl_
+  interval: 10m
+pruning:
+  keep_local:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+
+  keep_remote:
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+global:
+  logging:
+    - outlet:
+        type: "stdout"
+        time: true
+      level: "warn"
+      format: "human"

cmd/config/samples/sink.yml

@@ -0,0 +1,15 @@
+type: sink
+replication:
+  root_dataset: "pool2/backup_laptops"
+  serve:
+    type: tls
+    listen: "192.168.122.189:8888"
+    ca: "ca.pem"
+    cert: "cert.pem"
+    key: "key.pem"
+global:
+  logging:
+    - outlet:
+        type: "syslog"
+      level: "warn"
+      format: "human"