From 13dc63bd23f8802b2d110767397584b84f0ce846 Mon Sep 17 00:00:00 2001 From: Anton Schirg Date: Sun, 26 Aug 2018 16:44:34 +0200 Subject: [PATCH] build logger from new config --- cmd/config/config.go | 116 +++++++++++++++----------- cmd/config/config_test.go | 2 +- cmd/config/samples/push.yml | 50 ++++++------ cmd/config/samples/sink.yml | 27 +++--- cmd/config_logging.go | 159 ++++++++++++------------------------ cmd/daemon.go | 14 +++- 6 files changed, 170 insertions(+), 198 deletions(-) diff --git a/cmd/config/config.go b/cmd/config/config.go index 27af8cd..d7b52d9 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -1,54 +1,57 @@ package config import ( - "github.com/zrepl/yaml-config" "fmt" - "time" - "os" "github.com/pkg/errors" + "github.com/zrepl/yaml-config" "io/ioutil" + "os" + "time" ) -type NodeEnum struct { +type Config struct { + Jobs []JobEnum `yaml:"jobs"` + Global Global `yaml:"global"` +} + +type JobEnum struct { Ret interface{} } -type PushNode struct { - Type string `yaml:"type"` - Replication PushReplication `yaml:"replication"` - Snapshotting Snapshotting `yaml:"snapshotting"` - Pruning Pruning `yaml:"pruning"` - Global Global `yaml:"global"` +type PushJob struct { + Type string `yaml:"type"` + Replication PushReplication `yaml:"replication"` + Snapshotting Snapshotting `yaml:"snapshotting"` + Pruning Pruning `yaml:"pruning"` } -type SinkNode struct { - Type string `yaml:"type"` +type SinkJob struct { + Type string `yaml:"type"` Replication SinkReplication `yaml:"replication"` - Global Global `yaml:"global"` } type PushReplication struct { - Connect ConnectEnum `yaml:"connect"` + Connect ConnectEnum `yaml:"connect"` Filesystems map[string]bool `yaml:"filesystems"` } type SinkReplication struct { - RootDataset string `yaml:"root_dataset"` - Serve ServeEnum `yaml:"serve"` + RootDataset string `yaml:"root_dataset"` + Serve ServeEnum `yaml:"serve"` } type Snapshotting struct { - SnapshotPrefix string `yaml:"snapshot_prefix"` - Interval time.Duration `yaml:"interval"` + SnapshotPrefix string `yaml:"snapshot_prefix"` + Interval time.Duration `yaml:"interval"` } type Pruning struct { - KeepLocal []PruningEnum `yaml:"keep_local"` + KeepLocal []PruningEnum `yaml:"keep_local"` KeepRemote []PruningEnum `yaml:"keep_remote"` } type Global struct { - Logging []LoggingOutlet `yaml:"logging"` + Logging []LoggingOutletEnum `yaml:"logging"` } type ConnectEnum struct { @@ -56,16 +59,16 @@ type ConnectEnum struct { } type TCPConnect struct { - Type string `yaml:"type"` + Type string `yaml:"type"` Address string `yaml:"address"` } type TLSConnect struct { - Type string `yaml:"type"` + Type string `yaml:"type"` Address string `yaml:"address"` - Ca string `yaml:"ca"` - Cert string `yaml:"cert"` - Key string `yaml:"key"` + Ca string `yaml:"ca"` + Cert string `yaml:"cert"` + Key string `yaml:"key"` } type ServeEnum struct { @@ -73,17 +76,17 @@ type ServeEnum struct { } type TCPServe struct { - Type string `yaml:"type"` - Listen string `yaml:"listen"` + Type string `yaml:"type"` + Listen string `yaml:"listen"` Clients map[string]string `yaml:"clients"` } type TLSServe struct { - Type string `yaml:"type"` + Type string `yaml:"type"` Listen string `yaml:"listen"` - Ca string `yaml:"ca"` - Cert string `yaml:"cert"` - Key string `yaml:"key"` + Ca string `yaml:"ca"` + Cert string `yaml:"cert"` + Key string `yaml:"key"` } type PruningEnum struct { @@ -95,8 +98,8 @@ type PruneKeepNotReplicated struct { } type PruneKeepLastN struct { - Type string `yaml:"type"` - Count int `yaml:"count"` + Type string `yaml:"type"` + Count int `yaml:"count"` } type PruneGrid struct { @@ -104,24 +107,38 @@ type PruneGrid struct { Grid string `yaml:"grid"` } -type LoggingOutlet struct { - Outlet LoggingOutletEnum `yaml:"outlet"` - Level string `yaml:"level"` - Format string `yaml:"format"` -} - type LoggingOutletEnum struct { Ret interface{} } +type LoggingOutletCommon struct { + Type string `yaml:"type"` + Level string `yaml:"level"` + Format string `yaml:"format"` +} + type StdoutLoggingOutlet struct { - Type string `yaml:"type"` - Time bool `yaml:"time"` + LoggingOutletCommon `yaml:",inline"` + Time bool `yaml:"time"` } type SyslogLoggingOutlet struct { - Type string `yaml:"type"` - RetryInterval time.Duration `yaml:"retry_interval"` + LoggingOutletCommon `yaml:",inline"` + RetryInterval time.Duration `yaml:"retry_interval"` +} + +type TCPLoggingOutlet struct { + LoggingOutletCommon `yaml:",inline"` + Address string `yaml:"address"` //TODO required + Net string `yaml:"net"` //TODO default tcp + RetryInterval time.Duration `yaml:"retry_interval"` + TLS *TCPLoggingOutletTLS +} + +type TCPLoggingOutletTLS struct { + CA string `yaml:"ca"` + Cert string `yaml:"cert"` + Key string `yaml:"key"` } func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) { @@ -145,10 +162,10 @@ func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{} return v, nil } -func (t *NodeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { +func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { t.Ret, err = enumUnmarshal(u, map[string]interface{}{ - "push": &PushNode{}, - "sink": &SinkNode{}, + "push": &PushJob{}, + "sink": &SinkJob{}, }) return } @@ -172,8 +189,8 @@ func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { t.Ret, err = enumUnmarshal(u, map[string]interface{}{ "not_replicated": &PruneKeepNotReplicated{}, - "last_n": &PruneKeepLastN{}, - "grid": &PruneGrid{}, + "last_n": &PruneKeepLastN{}, + "grid": &PruneGrid{}, }) return } @@ -182,6 +199,7 @@ func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err t.Ret, err = enumUnmarshal(u, map[string]interface{}{ "stdout": &StdoutLoggingOutlet{}, "syslog": &SyslogLoggingOutlet{}, + "tcp": &TCPLoggingOutlet{}, }) return } @@ -191,7 +209,7 @@ var ConfigFileDefaultLocations = []string{ "/usr/local/etc/zrepl/zrepl.yml", } -func ParseConfig(path string) (i NodeEnum, err error) { +func ParseConfig(path string) (i Config, err error) { if path == "" { // Try default locations diff --git a/cmd/config/config_test.go b/cmd/config/config_test.go index caab41a..ef4ca0c 100644 --- a/cmd/config/config_test.go +++ b/cmd/config/config_test.go @@ -1,9 +1,9 @@ package config import ( - "testing" "github.com/kr/pretty" "path/filepath" + "testing" ) func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) { diff --git a/cmd/config/samples/push.yml b/cmd/config/samples/push.yml index 9bec4d6..5966465 100644 --- a/cmd/config/samples/push.yml +++ b/cmd/config/samples/push.yml @@ -1,30 +1,30 @@ -type: push -replication: - connect: - type: tcp - address: "backup-server.foo.bar:8888" - filesystems: { - "<": true, - "tmp": false - } -snapshotting: - snapshot_prefix: zrepl_ - interval: 10m -pruning: - keep_local: - - type: not_replicated - - type: last_n - count: 10 - - type: grid - grid: 1x1h(keep=all) | 24x1h | 14x1d +jobs: + - type: push + replication: + connect: + type: tcp + address: "backup-server.foo.bar:8888" + filesystems: { + "<": true, + "tmp": false + } + snapshotting: + snapshot_prefix: zrepl_ + interval: 10m + pruning: + keep_local: + - type: not_replicated + - type: last_n + count: 10 + - type: grid + grid: 1x1h(keep=all) | 24x1h | 14x1d - keep_remote: - - type: grid - grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d + keep_remote: + - type: grid + grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d global: logging: - - outlet: - type: "stdout" - time: true + - type: "stdout" + time: true level: "warn" format: "human" diff --git a/cmd/config/samples/sink.yml b/cmd/config/samples/sink.yml index e55236e..95f4500 100644 --- a/cmd/config/samples/sink.yml +++ b/cmd/config/samples/sink.yml @@ -1,15 +1,20 @@ -type: sink -replication: - root_dataset: "pool2/backup_laptops" - serve: - type: tls - listen: "192.168.122.189:8888" - ca: "ca.pem" - cert: "cert.pem" - key: "key.pem" +jobs: + - type: sink + replication: + root_dataset: "pool2/backup_laptops" + serve: + type: tls + listen: "192.168.122.189:8888" + ca: "ca.pem" + cert: "cert.pem" + key: "key.pem" global: logging: - - outlet: - type: "syslog" + - type: "tcp" + address: "123.123.123.123:1234" + tls: + ca: "ca.pem" + cert: "cert.pem" + key: "key.pem" level: "warn" format: "human" diff --git a/cmd/config_logging.go b/cmd/config_logging.go index 200f365..4f18f0a 100644 --- a/cmd/config_logging.go +++ b/cmd/config_logging.go @@ -4,12 +4,11 @@ import ( "crypto/tls" "crypto/x509" "github.com/mattn/go-isatty" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" + "github.com/zrepl/zrepl/cmd/config" "github.com/zrepl/zrepl/cmd/tlsconf" "github.com/zrepl/zrepl/logger" "os" - "time" ) type LoggingConfig struct { @@ -26,16 +25,12 @@ const ( MetadataAll MetadataFlags = ^0 ) -func parseLogging(i interface{}) (c *LoggingConfig, err error) { +func parseLogging(in []config.LoggingOutletEnum) (c *LoggingConfig, err error) { c = &LoggingConfig{} c.Outlets = logger.NewOutlets() - var asList []interface{} - if err = mapstructure.Decode(i, &asList); err != nil { - return nil, errors.Wrap(err, "mapstructure error") - } - if len(asList) == 0 { + if len(in) == 0 { // Default config out := WriterOutlet{&HumanFormatter{}, os.Stdout} c.Outlets.Add(out, logger.Warn) @@ -43,7 +38,7 @@ func parseLogging(i interface{}) (c *LoggingConfig, err error) { } var syslogOutlets, stdoutOutlets int - for lei, le := range asList { + for lei, le := range in { outlet, minLevel, err := parseOutlet(le) if err != nil { @@ -95,56 +90,52 @@ func parseLogFormat(i interface{}) (f EntryFormatter, err error) { } -func parseOutlet(i interface{}) (o logger.Outlet, level logger.Level, err error) { +func parseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) { - var in struct { - Outlet string - Level string - Format string - } - if err = mapstructure.Decode(i, &in); err != nil { - err = errors.Wrap(err, "mapstructure error") - return - } - if in.Outlet == "" || in.Level == "" || in.Format == "" { - err = errors.Errorf("must specify 'outlet', 'level' and 'format' field") - return + parseCommon := func(common config.LoggingOutletCommon) (logger.Level, EntryFormatter, error) { + if common.Level == "" || common.Format == "" { + return 0, nil, errors.Errorf("must specify 'level' and 'format' field") + } + + minLevel, err := logger.ParseLevel(common.Level) + if err != nil { + return 0, nil, errors.Wrap(err, "cannot parse 'level' field") + } + formatter, err := parseLogFormat(common.Format) + if err != nil { + return 0, nil, errors.Wrap(err, "cannot parse 'formatter' field") + } + return minLevel, formatter, nil } - minLevel, err := logger.ParseLevel(in.Level) - if err != nil { - err = errors.Wrap(err, "cannot parse 'level' field") - return - } - formatter, err := parseLogFormat(in.Format) - if err != nil { - err = errors.Wrap(err, "cannot parse") - return - } + var f EntryFormatter - switch in.Outlet { - case "stdout": - o, err = parseStdoutOutlet(i, formatter) - case "tcp": - o, err = parseTCPOutlet(i, formatter) - case "syslog": - o, err = parseSyslogOutlet(i, formatter) + switch v := in.Ret.(type) { + case config.StdoutLoggingOutlet: + level, f, err = parseCommon(v.LoggingOutletCommon) + if err != nil { + break + } + o, err = parseStdoutOutlet(v, f) + case config.TCPLoggingOutlet: + level, f, err = parseCommon(v.LoggingOutletCommon) + if err != nil { + break + } + o, err = parseTCPOutlet(v, f) + case config.SyslogLoggingOutlet: + level, f, err = parseCommon(v.LoggingOutletCommon) + if err != nil { + break + } + o, err = parseSyslogOutlet(v, f) default: - err = errors.Errorf("unknown outlet type '%s'", in.Outlet) + panic(v) } - return o, minLevel, err - + return o, level, err } -func parseStdoutOutlet(i interface{}, formatter EntryFormatter) (WriterOutlet, error) { - - var in struct { - Time bool - } - if err := mapstructure.Decode(i, &in); err != nil { - return WriterOutlet{}, errors.Wrap(err, "invalid structure for stdout outlet") - } - +func parseStdoutOutlet(in config.StdoutLoggingOutlet, formatter EntryFormatter) (WriterOutlet, error) { flags := MetadataAll writer := os.Stdout if !isatty.IsTerminal(writer.Fd()) && !in.Time { @@ -158,54 +149,22 @@ func parseStdoutOutlet(i interface{}, formatter EntryFormatter) (WriterOutlet, e }, nil } -func parseTCPOutlet(i interface{}, formatter EntryFormatter) (out *TCPOutlet, err error) { - - var in struct { - Net string - Address string - RetryInterval string `mapstructure:"retry_interval"` - TLS map[string]interface{} - } - if err = mapstructure.Decode(i, &in); err != nil { - return nil, errors.Wrap(err, "mapstructure error") - } - - retryInterval, err := time.ParseDuration(in.RetryInterval) - if err != nil { - return nil, errors.Wrap(err, "cannot parse 'retry_interval'") - } - - if len(in.Net) == 0 { - return nil, errors.New("field 'net' must not be empty") - } - if len(in.Address) == 0 { - return nil, errors.New("field 'address' must not be empty") - } - +func parseTCPOutlet(in config.TCPLoggingOutlet, formatter EntryFormatter) (out *TCPOutlet, err error) { var tlsConfig *tls.Config if in.TLS != nil { - tlsConfig, err = func(m map[string]interface{}, host string) (*tls.Config, error) { - var in struct { - CA string - Cert string - Key string - } - if err := mapstructure.Decode(m, &in); err != nil { - return nil, errors.Wrap(err, "mapstructure error") - } - - clientCert, err := tls.LoadX509KeyPair(in.Cert, in.Key) + tlsConfig, err = func(m *config.TCPLoggingOutletTLS, host string) (*tls.Config, error) { + clientCert, err := tls.LoadX509KeyPair(m.Cert, m.Key) if err != nil { return nil, errors.Wrap(err, "cannot load client cert") } var rootCAs *x509.CertPool - if in.CA == "" { + if m.CA == "" { if rootCAs, err = x509.SystemCertPool(); err != nil { return nil, errors.Wrap(err, "cannot open system cert pool") } } else { - rootCAs, err = tlsconf.ParseCAFile(in.CA) + rootCAs, err = tlsconf.ParseCAFile(m.CA) if err != nil { return nil, errors.Wrap(err, "cannot parse CA cert") } @@ -222,30 +181,14 @@ func parseTCPOutlet(i interface{}, formatter EntryFormatter) (out *TCPOutlet, er } formatter.SetMetadataFlags(MetadataAll) - return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, retryInterval), nil + return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, in.RetryInterval), nil } -func parseSyslogOutlet(i interface{}, formatter EntryFormatter) (out *SyslogOutlet, err error) { - - var in struct { - RetryInterval string `mapstructure:"retry_interval"` - } - if err = mapstructure.Decode(i, &in); err != nil { - return nil, errors.Wrap(err, "mapstructure error") - } - +func parseSyslogOutlet(in config.SyslogLoggingOutlet, formatter EntryFormatter) (out *SyslogOutlet, err error) { out = &SyslogOutlet{} out.Formatter = formatter out.Formatter.SetMetadataFlags(MetadataNone) - - out.RetryInterval = 0 // default to 0 as we assume local syslog will just work - if in.RetryInterval != "" { - out.RetryInterval, err = time.ParseDuration(in.RetryInterval) - if err != nil { - return nil, errors.Wrap(err, "cannot parse 'retry_interval'") - } - } - - return + out.RetryInterval = in.RetryInterval + return out, nil } diff --git a/cmd/daemon.go b/cmd/daemon.go index 94a32d4..3fd8d2f 100644 --- a/cmd/daemon.go +++ b/cmd/daemon.go @@ -4,14 +4,14 @@ import ( "context" "fmt" "github.com/spf13/cobra" + "github.com/zrepl/zrepl/cmd/config" + "github.com/zrepl/zrepl/logger" "os" "os/signal" - "syscall" "time" "github.com/zrepl/zrepl/cmd/daemon" "github.com/zrepl/zrepl/cmd/daemon/job" - "github.com/zrepl/zrepl/logger" ) // daemonCmd represents the daemon command @@ -75,13 +75,19 @@ func (a daemonJobAdaptor) Status() interface{} { return nil } func doDaemon(cmd *cobra.Command, args []string) { - conf, err := ParseConfig(rootArgs.configFile) + conf, err := config.ParseConfig(rootArgs.configFile) if err != nil { fmt.Fprintf(os.Stderr, "error parsing config: %s\n", err) os.Exit(1) } - log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second) + outlets, err := parseLogging(conf.Global.Logging) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to generate logger: %s\n", err) + return + } + log := logger.NewLogger(outlets.Outlets, 1*time.Second) + ctx := WithLogger(context.Background(), log) daemonJobs := make([]job.Job, 0, len(conf.Jobs))