config for pull and source

cmd/config/config.go

@@ -21,17 +21,39 @@ type JobEnum struct {
 }
 
 type PushJob struct {
-	Type         string          `yaml:"type"`
+	Type         string                `yaml:"type"`
-	Replication  PushReplication `yaml:"replication"`
+	Name         string                `yaml:"name"`
-	Snapshotting Snapshotting    `yaml:"snapshotting"`
+	Replication  PushReplication       `yaml:"replication"`
-	Pruning      Pruning         `yaml:"pruning"`
+	Snapshotting Snapshotting          `yaml:"snapshotting"`
+	Pruning      PruningSenderReceiver `yaml:"pruning"`
 }
 
 type SinkJob struct {
 	Type        string          `yaml:"type"`
+	Name        string          `yaml:"name"`
 	Replication SinkReplication `yaml:"replication"`
 }
 
+type PullJob struct {
+	Type        string                `yaml:"type"`
+	Name        string                `yaml:"name"`
+	Replication PullReplication       `yaml:"replication"`
+	Pruning     PruningSenderReceiver `yaml:"pruning"`
+}
+
+type PullReplication struct {
+	Connect     ConnectEnum `yaml:"connect"`
+	RootDataset string      `yaml:"root_dataset"`
+}
+
+type SourceJob struct {
+	Type         string            `yaml:"type"`
+	Name         string            `yaml:"name"`
+	Replication  SourceReplication `yaml:"replication"`
+	Snapshotting Snapshotting      `yaml:"snapshotting"`
+	Pruning      PruningLocal      `yaml:"pruning"`
+}
+
 type PushReplication struct {
 	Connect     ConnectEnum     `yaml:"connect"`
 	Filesystems map[string]bool `yaml:"filesystems"`
@@ -42,19 +64,30 @@ type SinkReplication struct {
 	Serve       ServeEnum `yaml:"serve"`
 }
 
+type SourceReplication struct {
+	Serve       ServeEnum       `yaml:"serve"`
+	Filesystems map[string]bool `yaml:"filesystems"`
+}
+
 type Snapshotting struct {
 	SnapshotPrefix string        `yaml:"snapshot_prefix"`
 	Interval       time.Duration `yaml:"interval"`
 }
 
-type Pruning struct {
+type PruningSenderReceiver struct {
-	KeepLocal  []PruningEnum `yaml:"keep_local"`
+	KeepSender   []PruningEnum `yaml:"keep_sender"`
-	KeepRemote []PruningEnum `yaml:"keep_remote"`
+	KeepReceiver []PruningEnum `yaml:"keep_receiver"`
+}
+
+type PruningLocal struct {
+	Keep []PruningEnum `yaml:"keep"`
 }
 
 type Global struct {
 	Logging    []LoggingOutletEnum `yaml:"logging"`
 	Monitoring []MonitoringEnum    `yaml:"monitoring,optional"`
+	Control    GlobalControl       `yaml:"control"`
+	Serve      GlobalServe         `yaml:"serve"`
 }
 
 type ConnectEnum struct {
@@ -127,9 +160,9 @@ type SyslogLoggingOutlet struct {
 
 type TCPLoggingOutlet struct {
 	LoggingOutletCommon `yaml:",inline"`
-	Address             string        `yaml:"address"`
+	Address             string               `yaml:"address"`
-	Net                 string        `yaml:"net,default=tcp"`
+	Net                 string               `yaml:"net,default=tcp"`
-	RetryInterval       time.Duration `yaml:"retry_interval"`
+	RetryInterval       time.Duration        `yaml:"retry_interval"`
 	TLS                 *TCPLoggingOutletTLS `yaml:"tls,optional"`
 }
 
@@ -148,6 +181,18 @@ type PrometheusMonitoring struct {
 	Listen string `yaml:"listen"`
 }
 
+type GlobalControl struct {
+	SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
+}
+
+type GlobalServe struct {
+	StdinServer GlobalStdinServer `yaml:"stdinserver"`
+}
+
+type GlobalStdinServer struct {
+	SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
+}
+
 func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
 	var in struct {
 		Type string
@@ -171,8 +216,10 @@ func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}
 
 func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
 	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"push": &PushJob{},
+		"push":   &PushJob{},
-		"sink": &SinkJob{},
+		"sink":   &SinkJob{},
+		"pull":   &PullJob{},
+		"source": &SourceJob{},
 	})
 	return
 }

cmd/config/config_test.go

@@ -14,13 +14,15 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
 
 	for _, p := range paths {
 
-		c, err := ParseConfig(p)
+		t.Run(p, func(t *testing.T) {
-		if err != nil {
+			c, err := ParseConfig(p)
-			t.Errorf("error parsing %s:\n%+v", p, err)
+			if err != nil {
-		}
+				t.Errorf("error parsing %s:\n%+v", p, err)
+			}
 
-		t.Logf("file: %s", p)
+			t.Logf("file: %s", p)
-		t.Log(pretty.Sprint(c))
+			t.Log(pretty.Sprint(c))
+		})
 
 	}
 

cmd/config/samples/pull.yml

@@ -0,0 +1,39 @@
+jobs:
+
+- name: pull_servers
+  type: pull
+  replication:
+    connect:
+      type: tls
+      address: "server1.foo.bar:8888"
+      ca: /certs/ca.crt
+      cert: /certs/cert.crt
+      key: /certs/key.pem
+    root_dataset: "pool2/backup_servers"
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      keep_bookmarks: all
+    keep_receiver:
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+      keep_bookmarks: all
+
+global:
+  logging:
+  - type: "stdout"
+    time: true
+    level: "warn"
+    format: "human"
+  monitoring:
+  - type: "prometheus"
+    listen: ":9091"
+  control:
+    sockpath: /var/run/zrepl/control
+  serve:
+    stdinserver:
+      sockdir: /var/run/zrepl/stdinserver

cmd/config/samples/push.yml

@@ -1,5 +1,6 @@
 jobs:
   - type: push
+    name: "push"
     replication:
       connect:
         type: tcp
@@ -12,7 +13,7 @@ jobs:
       snapshot_prefix: zrepl_
       interval: 10m
     pruning:
-      keep_local:
+      keep_sender:
         - type: not_replicated
         - type: last_n
           count: 10
@@ -20,7 +21,7 @@ jobs:
           grid: 1x1h(keep=all) | 24x1h | 14x1d
           keep_bookmarks: all
 
-      keep_remote:
+      keep_receiver:
         - type: grid
           grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
           keep_bookmarks: all
@@ -32,4 +33,9 @@ global:
       format: "human"
   monitoring:
     - type: "prometheus"
-      listen: ":9091"
+      listen: ":9091"
+  control:
+    sockpath: /var/run/zrepl/control
+  serve:
+    stdinserver:
+      sockdir: /var/run/zrepl/stdinserver

cmd/config/samples/sink.yml

@@ -1,5 +1,6 @@
 jobs:
   - type: sink
+    name: "laptop_sink"
     replication:
       root_dataset: "pool2/backup_laptops"
       serve:
@@ -19,3 +20,8 @@ global:
         key: "key.pem"
       level: "warn"
       format: "human"
+  control:
+    sockpath: /var/run/zrepl/control
+  serve:
+    stdinserver:
+      sockdir: /var/run/zrepl/stdinserver

cmd/config/samples/source.yml

@@ -0,0 +1,40 @@
+jobs:
+- name: pull_source
+  type: source
+  replication:
+    serve:
+      type: tcp
+      listen: "0.0.0.0:8888"
+      clients: {
+        "192.168.122.123" : "client1"
+      }
+    filesystems: {
+      "<": true,
+      "secret": false
+    }
+  snapshotting:
+    snapshot_prefix: zrepl_
+    interval: 10m
+  pruning:
+    keep:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      keep_bookmarks: all
+
+global:
+  logging:
+  - type: "stdout"
+    time: true
+    level: "warn"
+    format: "human"
+  monitoring:
+  - type: "prometheus"
+    listen: ":9091"
+  control:
+    sockpath: /var/run/zrepl/control
+  serve:
+    stdinserver:
+      sockdir: /var/run/zrepl/stdinserver