package config import ( "fmt" "github.com/pkg/errors" "github.com/zrepl/yaml-config" "io/ioutil" "os" "regexp" "strconv" "time" ) type Config struct { Jobs []JobEnum `yaml:"jobs"` Global Global `yaml:"global"` } type JobEnum struct { Ret interface{} } type PushJob struct { Type string `yaml:"type"` Name string `yaml:"name"` Replication PushReplication `yaml:"replication"` Snapshotting Snapshotting `yaml:"snapshotting"` Pruning PruningSenderReceiver `yaml:"pruning"` Debug JobDebugSettings `yaml:"debug,optional"` } type SinkJob struct { Type string `yaml:"type"` Name string `yaml:"name"` Replication SinkReplication `yaml:"replication"` Debug JobDebugSettings `yaml:"debug,optional"` } type PullJob struct { Type string `yaml:"type"` Name string `yaml:"name"` Replication PullReplication `yaml:"replication"` Pruning PruningSenderReceiver `yaml:"pruning"` Debug JobDebugSettings `yaml:"debug,optional"` } type PullReplication struct { Connect ConnectEnum `yaml:"connect"` RootDataset string `yaml:"root_dataset"` Interval time.Duration `yaml:"interval,positive"` } type SourceJob struct { Type string `yaml:"type"` Name string `yaml:"name"` Replication SourceReplication `yaml:"replication"` Snapshotting Snapshotting `yaml:"snapshotting"` Pruning PruningLocal `yaml:"pruning"` Debug JobDebugSettings `yaml:"debug,optional"` } type LocalJob struct { Type string `yaml:"type"` Name string `yaml:"name"` Replication LocalReplication `yaml:"replication"` Snapshotting Snapshotting `yaml:"snapshotting"` Pruning PruningSenderReceiver `yaml:"pruning"` Debug JobDebugSettings `yaml:"debug,optional"` } type PushReplication struct { Connect ConnectEnum `yaml:"connect"` Filesystems map[string]bool `yaml:"filesystems"` } type SinkReplication struct { RootDataset string `yaml:"root_dataset"` Serve ServeEnum `yaml:"serve"` } type SourceReplication struct { Serve ServeEnum `yaml:"serve"` Filesystems map[string]bool `yaml:"filesystems"` } type LocalReplication struct { Filesystems map[string]bool `yaml:"filesystems"` RootDataset string `yaml:"root_dataset"` } type Snapshotting struct { SnapshotPrefix string `yaml:"snapshot_prefix"` Interval time.Duration `yaml:"interval,positive"` } type PruningSenderReceiver struct { KeepSender []PruningEnum `yaml:"keep_sender"` KeepReceiver []PruningEnum `yaml:"keep_receiver"` } type PruningLocal struct { Keep []PruningEnum `yaml:"keep"` } type Global struct { Logging []LoggingOutletEnum `yaml:"logging"` Monitoring []MonitoringEnum `yaml:"monitoring,optional"` Control GlobalControl `yaml:"control"` Serve GlobalServe `yaml:"serve"` } type ConnectEnum struct { Ret interface{} } type TCPConnect struct { Type string `yaml:"type"` Address string `yaml:"address"` DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"` } type TLSConnect struct { Type string `yaml:"type"` Address string `yaml:"address"` Ca string `yaml:"ca"` Cert string `yaml:"cert"` Key string `yaml:"key"` ServerCN string `yaml:"server_cn"` DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"` } type SSHStdinserverConnect struct { Type string `yaml:"type"` Host string `yaml:"host"` User string `yaml:"user"` Port uint16 `yaml:"port"` IdentityFile string `yaml:"identity_file"` TransportOpenCommand []string `yaml:"transport_open_command,optional"` //TODO unused SSHCommand string `yaml:"ssh_command,optional"` //TODO unused Options []string `yaml:"options"` DialTimeout time.Duration `yaml:"dial_timeout,positive,default=10s"` } type ServeEnum struct { Ret interface{} } type TCPServe struct { Type string `yaml:"type"` Listen string `yaml:"listen"` Clients map[string]string `yaml:"clients"` } type TLSServe struct { Type string `yaml:"type"` Listen string `yaml:"listen"` Ca string `yaml:"ca"` Cert string `yaml:"cert"` Key string `yaml:"key"` ClientCN string `yaml:"client_cn"` HandshakeTimeout time.Duration `yaml:"handshake_timeout,positive,default=10s"` } type StdinserverServer struct { Type string `yaml:"type"` ClientIdentity string `yaml:"client_identity"` } type PruningEnum struct { Ret interface{} } type PruneKeepNotReplicated struct { Type string `yaml:"type"` } type PruneKeepLastN struct { Type string `yaml:"type"` Count int `yaml:"count"` } type PruneKeepRegex struct { // FIXME rename to KeepRegex Type string `yaml:"type"` Regex string `yaml:"prefix"` } type LoggingOutletEnum struct { Ret interface{} } type LoggingOutletCommon struct { Type string `yaml:"type"` Level string `yaml:"level"` Format string `yaml:"format"` } type StdoutLoggingOutlet struct { LoggingOutletCommon `yaml:",inline"` Time bool `yaml:"time,default=true"` } type SyslogLoggingOutlet struct { LoggingOutletCommon `yaml:",inline"` RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"` } type TCPLoggingOutlet struct { LoggingOutletCommon `yaml:",inline"` Address string `yaml:"address"` Net string `yaml:"net,default=tcp"` RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"` TLS *TCPLoggingOutletTLS `yaml:"tls,optional"` } type TCPLoggingOutletTLS struct { CA string `yaml:"ca"` Cert string `yaml:"cert"` Key string `yaml:"key"` } type MonitoringEnum struct { Ret interface{} } type PrometheusMonitoring struct { Type string `yaml:"type"` Listen string `yaml:"listen"` } type GlobalControl struct { SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"` } type GlobalServe struct { StdinServer GlobalStdinServer `yaml:"stdinserver"` } type GlobalStdinServer struct { SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"` } type JobDebugSettings struct { Conn struct { ReadDump string `yaml:"read_dump"` WriteDump string `yaml:"write_dump"` } `yaml:"conn"` } func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) { var in struct { Type string } if err := u(&in, true); err != nil { return nil, err } if in.Type == "" { return nil, &yaml.TypeError{[]string{"must specify type"}} } v, ok := types[in.Type] if !ok { return nil, &yaml.TypeError{[]string{fmt.Sprintf("invalid type name %q", in.Type)}} } if err := u(v, false); err != nil { return nil, err } return v, nil } func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { t.Ret, err = enumUnmarshal(u, map[string]interface{}{ "push": &PushJob{}, "sink": &SinkJob{}, "pull": &PullJob{}, "source": &SourceJob{}, "local": &LocalJob{}, }) return } func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { t.Ret, err = enumUnmarshal(u, map[string]interface{}{ "tcp": &TCPConnect{}, "tls": &TLSConnect{}, "ssh+stdinserver": &SSHStdinserverConnect{}, }) return } func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { t.Ret, err = enumUnmarshal(u, map[string]interface{}{ "tcp": &TCPServe{}, "tls": &TLSServe{}, "stdinserver": &StdinserverServer{}, }) return } func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { t.Ret, err = enumUnmarshal(u, map[string]interface{}{ "not_replicated": &PruneKeepNotReplicated{}, "last_n": &PruneKeepLastN{}, "grid": &PruneGrid{}, "prefix": &PruneKeepRegex{}, }) return } func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { t.Ret, err = enumUnmarshal(u, map[string]interface{}{ "stdout": &StdoutLoggingOutlet{}, "syslog": &SyslogLoggingOutlet{}, "tcp": &TCPLoggingOutlet{}, }) return } func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { t.Ret, err = enumUnmarshal(u, map[string]interface{}{ "prometheus": &PrometheusMonitoring{}, }) return } var ConfigFileDefaultLocations = []string{ "/etc/zrepl/zrepl.yml", "/usr/local/etc/zrepl/zrepl.yml", } func ParseConfig(path string) (i Config, err error) { if path == "" { // Try default locations for _, l := range ConfigFileDefaultLocations { stat, statErr := os.Stat(l) if statErr != nil { continue } if !stat.Mode().IsRegular() { err = errors.Errorf("file at default location is not a regular file: %s", l) return } path = l break } } var bytes []byte if bytes, err = ioutil.ReadFile(path); err != nil { return } if err = yaml.UnmarshalStrict(bytes, &i); err != nil { return } return } var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`) func parsePostitiveDuration(e string) (d time.Duration, err error) { comps := durationStringRegex.FindStringSubmatch(e) if len(comps) != 3 { err = fmt.Errorf("does not match regex: %s %#v", e, comps) return } durationFactor, err := strconv.ParseInt(comps[1], 10, 64) if err != nil { return 0, err } if durationFactor <= 0 { return 0, errors.New("duration must be positive integer") } var durationUnit time.Duration switch comps[2] { case "s": durationUnit = time.Second case "m": durationUnit = time.Minute case "h": durationUnit = time.Hour case "d": durationUnit = 24 * time.Hour case "w": durationUnit = 24 * 7 * time.Hour default: err = fmt.Errorf("contains unknown time unit '%s'", comps[2]) return } d = time.Duration(durationFactor) * durationUnit return }