From 48a08e4f4df0edaa233bcc090b143b53cdb5fa36 Mon Sep 17 00:00:00 2001 From: Anton Schirg Date: Sun, 26 Aug 2018 23:11:50 +0200 Subject: [PATCH] config for pull and source --- cmd/config/config.go | 71 +++++++++++++++++++++++++++++------ cmd/config/config_test.go | 14 ++++--- cmd/config/samples/pull.yml | 39 +++++++++++++++++++ cmd/config/samples/push.yml | 12 ++++-- cmd/config/samples/sink.yml | 6 +++ cmd/config/samples/source.yml | 40 ++++++++++++++++++++ 6 files changed, 161 insertions(+), 21 deletions(-) create mode 100644 cmd/config/samples/pull.yml create mode 100644 cmd/config/samples/source.yml diff --git a/cmd/config/config.go b/cmd/config/config.go index 6f79352..e45637a 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -21,17 +21,39 @@ type JobEnum struct { } type PushJob struct { - Type string `yaml:"type"` - Replication PushReplication `yaml:"replication"` - Snapshotting Snapshotting `yaml:"snapshotting"` - Pruning Pruning `yaml:"pruning"` + Type string `yaml:"type"` + Name string `yaml:"name"` + Replication PushReplication `yaml:"replication"` + Snapshotting Snapshotting `yaml:"snapshotting"` + Pruning PruningSenderReceiver `yaml:"pruning"` } type SinkJob struct { Type string `yaml:"type"` + Name string `yaml:"name"` Replication SinkReplication `yaml:"replication"` } +type PullJob struct { + Type string `yaml:"type"` + Name string `yaml:"name"` + Replication PullReplication `yaml:"replication"` + Pruning PruningSenderReceiver `yaml:"pruning"` +} + +type PullReplication struct { + Connect ConnectEnum `yaml:"connect"` + RootDataset string `yaml:"root_dataset"` +} + +type SourceJob struct { + Type string `yaml:"type"` + Name string `yaml:"name"` + Replication SourceReplication `yaml:"replication"` + Snapshotting Snapshotting `yaml:"snapshotting"` + Pruning PruningLocal `yaml:"pruning"` +} + type PushReplication struct { Connect ConnectEnum `yaml:"connect"` Filesystems map[string]bool `yaml:"filesystems"` @@ -42,19 +64,30 @@ type SinkReplication struct { Serve ServeEnum `yaml:"serve"` } +type SourceReplication struct { + Serve ServeEnum `yaml:"serve"` + Filesystems map[string]bool `yaml:"filesystems"` +} + type Snapshotting struct { SnapshotPrefix string `yaml:"snapshot_prefix"` Interval time.Duration `yaml:"interval"` } -type Pruning struct { - KeepLocal []PruningEnum `yaml:"keep_local"` - KeepRemote []PruningEnum `yaml:"keep_remote"` +type PruningSenderReceiver struct { + KeepSender []PruningEnum `yaml:"keep_sender"` + KeepReceiver []PruningEnum `yaml:"keep_receiver"` +} + +type PruningLocal struct { + Keep []PruningEnum `yaml:"keep"` } type Global struct { Logging []LoggingOutletEnum `yaml:"logging"` Monitoring []MonitoringEnum `yaml:"monitoring,optional"` + Control GlobalControl `yaml:"control"` + Serve GlobalServe `yaml:"serve"` } type ConnectEnum struct { @@ -127,9 +160,9 @@ type SyslogLoggingOutlet struct { type TCPLoggingOutlet struct { LoggingOutletCommon `yaml:",inline"` - Address string `yaml:"address"` - Net string `yaml:"net,default=tcp"` - RetryInterval time.Duration `yaml:"retry_interval"` + Address string `yaml:"address"` + Net string `yaml:"net,default=tcp"` + RetryInterval time.Duration `yaml:"retry_interval"` TLS *TCPLoggingOutletTLS `yaml:"tls,optional"` } @@ -148,6 +181,18 @@ type PrometheusMonitoring struct { Listen string `yaml:"listen"` } +type GlobalControl struct { + SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"` +} + +type GlobalServe struct { + StdinServer GlobalStdinServer `yaml:"stdinserver"` +} + +type GlobalStdinServer struct { + SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"` +} + func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) { var in struct { Type string @@ -171,8 +216,10 @@ func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{} func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { t.Ret, err = enumUnmarshal(u, map[string]interface{}{ - "push": &PushJob{}, - "sink": &SinkJob{}, + "push": &PushJob{}, + "sink": &SinkJob{}, + "pull": &PullJob{}, + "source": &SourceJob{}, }) return } diff --git a/cmd/config/config_test.go b/cmd/config/config_test.go index ef4ca0c..ad975d0 100644 --- a/cmd/config/config_test.go +++ b/cmd/config/config_test.go @@ -14,13 +14,15 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) { for _, p := range paths { - c, err := ParseConfig(p) - if err != nil { - t.Errorf("error parsing %s:\n%+v", p, err) - } + t.Run(p, func(t *testing.T) { + c, err := ParseConfig(p) + if err != nil { + t.Errorf("error parsing %s:\n%+v", p, err) + } - t.Logf("file: %s", p) - t.Log(pretty.Sprint(c)) + t.Logf("file: %s", p) + t.Log(pretty.Sprint(c)) + }) } diff --git a/cmd/config/samples/pull.yml b/cmd/config/samples/pull.yml new file mode 100644 index 0000000..dce4706 --- /dev/null +++ b/cmd/config/samples/pull.yml @@ -0,0 +1,39 @@ +jobs: + +- name: pull_servers + type: pull + replication: + connect: + type: tls + address: "server1.foo.bar:8888" + ca: /certs/ca.crt + cert: /certs/cert.crt + key: /certs/key.pem + root_dataset: "pool2/backup_servers" + pruning: + keep_sender: + - type: not_replicated + - type: last_n + count: 10 + - type: grid + grid: 1x1h(keep=all) | 24x1h | 14x1d + keep_bookmarks: all + keep_receiver: + - type: grid + grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d + keep_bookmarks: all + +global: + logging: + - type: "stdout" + time: true + level: "warn" + format: "human" + monitoring: + - type: "prometheus" + listen: ":9091" + control: + sockpath: /var/run/zrepl/control + serve: + stdinserver: + sockdir: /var/run/zrepl/stdinserver \ No newline at end of file diff --git a/cmd/config/samples/push.yml b/cmd/config/samples/push.yml index d12c57a..5765a87 100644 --- a/cmd/config/samples/push.yml +++ b/cmd/config/samples/push.yml @@ -1,5 +1,6 @@ jobs: - type: push + name: "push" replication: connect: type: tcp @@ -12,7 +13,7 @@ jobs: snapshot_prefix: zrepl_ interval: 10m pruning: - keep_local: + keep_sender: - type: not_replicated - type: last_n count: 10 @@ -20,7 +21,7 @@ jobs: grid: 1x1h(keep=all) | 24x1h | 14x1d keep_bookmarks: all - keep_remote: + keep_receiver: - type: grid grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d keep_bookmarks: all @@ -32,4 +33,9 @@ global: format: "human" monitoring: - type: "prometheus" - listen: ":9091" \ No newline at end of file + listen: ":9091" + control: + sockpath: /var/run/zrepl/control + serve: + stdinserver: + sockdir: /var/run/zrepl/stdinserver \ No newline at end of file diff --git a/cmd/config/samples/sink.yml b/cmd/config/samples/sink.yml index f1e550f..1803512 100644 --- a/cmd/config/samples/sink.yml +++ b/cmd/config/samples/sink.yml @@ -1,5 +1,6 @@ jobs: - type: sink + name: "laptop_sink" replication: root_dataset: "pool2/backup_laptops" serve: @@ -19,3 +20,8 @@ global: key: "key.pem" level: "warn" format: "human" + control: + sockpath: /var/run/zrepl/control + serve: + stdinserver: + sockdir: /var/run/zrepl/stdinserver \ No newline at end of file diff --git a/cmd/config/samples/source.yml b/cmd/config/samples/source.yml new file mode 100644 index 0000000..79a4847 --- /dev/null +++ b/cmd/config/samples/source.yml @@ -0,0 +1,40 @@ +jobs: +- name: pull_source + type: source + replication: + serve: + type: tcp + listen: "0.0.0.0:8888" + clients: { + "192.168.122.123" : "client1" + } + filesystems: { + "<": true, + "secret": false + } + snapshotting: + snapshot_prefix: zrepl_ + interval: 10m + pruning: + keep: + - type: not_replicated + - type: last_n + count: 10 + - type: grid + grid: 1x1h(keep=all) | 24x1h | 14x1d + keep_bookmarks: all + +global: + logging: + - type: "stdout" + time: true + level: "warn" + format: "human" + monitoring: + - type: "prometheus" + listen: ":9091" + control: + sockpath: /var/run/zrepl/control + serve: + stdinserver: + sockdir: /var/run/zrepl/stdinserver \ No newline at end of file