extern/zrepl
1
0
Fork 0
mirror of https://github.com/zrepl/zrepl.git synced 2025-05-29 22:30:49 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

build logger from new config

Browse Source
This commit is contained in:
Anton Schirg 2018-08-26 16:44:34 +02:00
parent 38bb78b642
commit 13dc63bd23
6 changed files with 170 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
cmd/config/config.go
View File

@ -1,54 +1,57 @@
package config package config
import ( import (
"github.com/zrepl/yaml-config"
"fmt" "fmt"
"time"
"os"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/zrepl/yaml-config"
"io/ioutil" "io/ioutil"
"os"
"time"
) )
type NodeEnum struct { type Config struct {
Jobs []JobEnum `yaml:"jobs"`
Global Global `yaml:"global"`
}
type JobEnum struct {
Ret interface{} Ret interface{}
} }
type PushNode struct { type PushJob struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Replication PushReplication `yaml:"replication"` Replication PushReplication `yaml:"replication"`
Snapshotting Snapshotting `yaml:"snapshotting"` Snapshotting Snapshotting `yaml:"snapshotting"`
Pruning Pruning `yaml:"pruning"` Pruning Pruning `yaml:"pruning"`
Global Global `yaml:"global"`
} }
type SinkNode struct { type SinkJob struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Replication SinkReplication `yaml:"replication"` Replication SinkReplication `yaml:"replication"`
Global Global `yaml:"global"`
} }
type PushReplication struct { type PushReplication struct {
Connect ConnectEnum `yaml:"connect"` Connect ConnectEnum `yaml:"connect"`
Filesystems map[string]bool `yaml:"filesystems"` Filesystems map[string]bool `yaml:"filesystems"`
} }
type SinkReplication struct { type SinkReplication struct {
RootDataset string `yaml:"root_dataset"` RootDataset string `yaml:"root_dataset"`
Serve ServeEnum `yaml:"serve"` Serve ServeEnum `yaml:"serve"`
} }
type Snapshotting struct { type Snapshotting struct {
SnapshotPrefix string `yaml:"snapshot_prefix"` SnapshotPrefix string `yaml:"snapshot_prefix"`
Interval time.Duration `yaml:"interval"` Interval time.Duration `yaml:"interval"`
} }
type Pruning struct { type Pruning struct {
KeepLocal []PruningEnum `yaml:"keep_local"` KeepLocal []PruningEnum `yaml:"keep_local"`
KeepRemote []PruningEnum `yaml:"keep_remote"` KeepRemote []PruningEnum `yaml:"keep_remote"`
} }
type Global struct { type Global struct {
Logging []LoggingOutlet `yaml:"logging"` Logging []LoggingOutletEnum `yaml:"logging"`
} }
type ConnectEnum struct { type ConnectEnum struct {
@ -56,16 +59,16 @@ type ConnectEnum struct {
} }
type TCPConnect struct { type TCPConnect struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Address string `yaml:"address"` Address string `yaml:"address"`
} }
type TLSConnect struct { type TLSConnect struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Address string `yaml:"address"` Address string `yaml:"address"`
Ca string `yaml:"ca"` Ca string `yaml:"ca"`
Cert string `yaml:"cert"` Cert string `yaml:"cert"`
Key string `yaml:"key"` Key string `yaml:"key"`
} }
type ServeEnum struct { type ServeEnum struct {
@ -73,17 +76,17 @@ type ServeEnum struct {
} }
type TCPServe struct { type TCPServe struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Listen string `yaml:"listen"` Listen string `yaml:"listen"`
Clients map[string]string `yaml:"clients"` Clients map[string]string `yaml:"clients"`
} }
type TLSServe struct { type TLSServe struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Listen string `yaml:"listen"` Listen string `yaml:"listen"`
Ca string `yaml:"ca"` Ca string `yaml:"ca"`
Cert string `yaml:"cert"` Cert string `yaml:"cert"`
Key string `yaml:"key"` Key string `yaml:"key"`
} }
type PruningEnum struct { type PruningEnum struct {
@ -95,8 +98,8 @@ type PruneKeepNotReplicated struct {
} }
type PruneKeepLastN struct { type PruneKeepLastN struct {
Type string `yaml:"type"` Type string `yaml:"type"`
Count int `yaml:"count"` Count int `yaml:"count"`
} }
type PruneGrid struct { type PruneGrid struct {
@ -104,24 +107,38 @@ type PruneGrid struct {
Grid string `yaml:"grid"` Grid string `yaml:"grid"`
} }
type LoggingOutlet struct {
Outlet LoggingOutletEnum `yaml:"outlet"`
Level string `yaml:"level"`
Format string `yaml:"format"`
}
type LoggingOutletEnum struct { type LoggingOutletEnum struct {
Ret interface{} Ret interface{}
} }
type LoggingOutletCommon struct {
Type string `yaml:"type"`
Level string `yaml:"level"`
Format string `yaml:"format"`
}
type StdoutLoggingOutlet struct { type StdoutLoggingOutlet struct {
Type string `yaml:"type"` LoggingOutletCommon `yaml:",inline"`
Time bool `yaml:"time"` Time bool `yaml:"time"`
} }
type SyslogLoggingOutlet struct { type SyslogLoggingOutlet struct {
Type string `yaml:"type"` LoggingOutletCommon `yaml:",inline"`
RetryInterval time.Duration `yaml:"retry_interval"` RetryInterval time.Duration `yaml:"retry_interval"`
}
type TCPLoggingOutlet struct {
LoggingOutletCommon `yaml:",inline"`
Address string `yaml:"address"` //TODO required
Net string `yaml:"net"` //TODO default tcp
RetryInterval time.Duration `yaml:"retry_interval"`
TLS *TCPLoggingOutletTLS
}
type TCPLoggingOutletTLS struct {
CA string `yaml:"ca"`
Cert string `yaml:"cert"`
Key string `yaml:"key"`
} }
func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) { func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
@ -145,10 +162,10 @@ func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}
return v, nil return v, nil
} }
func (t *NodeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{ t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"push": &PushNode{}, "push": &PushJob{},
"sink": &SinkNode{}, "sink": &SinkJob{},
}) })
return return
} }
@ -172,8 +189,8 @@ func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
t.Ret, err = enumUnmarshal(u, map[string]interface{}{ t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"not_replicated": &PruneKeepNotReplicated{}, "not_replicated": &PruneKeepNotReplicated{},
"last_n": &PruneKeepLastN{}, "last_n": &PruneKeepLastN{},
"grid": &PruneGrid{}, "grid": &PruneGrid{},
}) })
return return
} }
@ -182,6 +199,7 @@ func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err
t.Ret, err = enumUnmarshal(u, map[string]interface{}{ t.Ret, err = enumUnmarshal(u, map[string]interface{}{
"stdout": &StdoutLoggingOutlet{}, "stdout": &StdoutLoggingOutlet{},
"syslog": &SyslogLoggingOutlet{}, "syslog": &SyslogLoggingOutlet{},
"tcp": &TCPLoggingOutlet{},
}) })
return return
} }
@ -191,7 +209,7 @@ var ConfigFileDefaultLocations = []string{
"/usr/local/etc/zrepl/zrepl.yml", "/usr/local/etc/zrepl/zrepl.yml",
} }
func ParseConfig(path string) (i NodeEnum, err error) { func ParseConfig(path string) (i Config, err error) {
if path == "" { if path == "" {
// Try default locations // Try default locations

2
cmd/config/config_test.go
View File

@ -1,9 +1,9 @@
package config package config
import ( import (
"testing"
"github.com/kr/pretty" "github.com/kr/pretty"
"path/filepath" "path/filepath"
"testing"
) )
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) { func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {

50
cmd/config/samples/push.yml
View File

@ -1,30 +1,30 @@
type: push jobs:
replication: - type: push
connect: replication:
type: tcp connect:
address: "backup-server.foo.bar:8888" type: tcp
filesystems: { address: "backup-server.foo.bar:8888"
"<": true, filesystems: {
"tmp": false "<": true,
} "tmp": false
snapshotting: }
snapshot_prefix: zrepl_ snapshotting:
interval: 10m snapshot_prefix: zrepl_
pruning: interval: 10m
keep_local: pruning:
- type: not_replicated keep_local:
- type: last_n - type: not_replicated
count: 10 - type: last_n
- type: grid count: 10
grid: 1x1h(keep=all) | 24x1h | 14x1d - type: grid
grid: 1x1h(keep=all) | 24x1h | 14x1d
keep_remote: keep_remote:
- type: grid - type: grid
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
global: global:
logging: logging:
- outlet: - type: "stdout"
type: "stdout" time: true
time: true
level: "warn" level: "warn"
format: "human" format: "human"

27
cmd/config/samples/sink.yml
View File

@ -1,15 +1,20 @@
type: sink jobs:
replication: - type: sink
root_dataset: "pool2/backup_laptops" replication:
serve: root_dataset: "pool2/backup_laptops"
type: tls serve:
listen: "192.168.122.189:8888" type: tls
ca: "ca.pem" listen: "192.168.122.189:8888"
cert: "cert.pem" ca: "ca.pem"
key: "key.pem" cert: "cert.pem"
key: "key.pem"
global: global:
logging: logging:
- outlet: - type: "tcp"
type: "syslog" address: "123.123.123.123:1234"
tls:
ca: "ca.pem"
cert: "cert.pem"
key: "key.pem"
level: "warn" level: "warn"
format: "human" format: "human"

159
cmd/config_logging.go
View File

@ -4,12 +4,11 @@ import (
"crypto/tls" "crypto/tls"
"crypto/x509" "crypto/x509"
"github.com/mattn/go-isatty" "github.com/mattn/go-isatty"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/cmd/tlsconf" "github.com/zrepl/zrepl/cmd/tlsconf"
"github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/logger"
"os" "os"
"time"
) )
type LoggingConfig struct { type LoggingConfig struct {
@ -26,16 +25,12 @@ const (
MetadataAll MetadataFlags = ^0 MetadataAll MetadataFlags = ^0
) )
func parseLogging(i interface{}) (c *LoggingConfig, err error) { func parseLogging(in []config.LoggingOutletEnum) (c *LoggingConfig, err error) {
c = &LoggingConfig{} c = &LoggingConfig{}
c.Outlets = logger.NewOutlets() c.Outlets = logger.NewOutlets()
var asList []interface{} if len(in) == 0 {
if err = mapstructure.Decode(i, &asList); err != nil {
return nil, errors.Wrap(err, "mapstructure error")
}
if len(asList) == 0 {
// Default config // Default config
out := WriterOutlet{&HumanFormatter{}, os.Stdout} out := WriterOutlet{&HumanFormatter{}, os.Stdout}
c.Outlets.Add(out, logger.Warn) c.Outlets.Add(out, logger.Warn)
@ -43,7 +38,7 @@ func parseLogging(i interface{}) (c *LoggingConfig, err error) {
} }
var syslogOutlets, stdoutOutlets int var syslogOutlets, stdoutOutlets int
for lei, le := range asList { for lei, le := range in {
outlet, minLevel, err := parseOutlet(le) outlet, minLevel, err := parseOutlet(le)
if err != nil { if err != nil {
@ -95,56 +90,52 @@ func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
} }
func parseOutlet(i interface{}) (o logger.Outlet, level logger.Level, err error) { func parseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) {
var in struct { parseCommon := func(common config.LoggingOutletCommon) (logger.Level, EntryFormatter, error) {
Outlet string if common.Level == "" || common.Format == "" {
Level string return 0, nil, errors.Errorf("must specify 'level' and 'format' field")
Format string }
}
if err = mapstructure.Decode(i, &in); err != nil { minLevel, err := logger.ParseLevel(common.Level)
err = errors.Wrap(err, "mapstructure error") if err != nil {
return return 0, nil, errors.Wrap(err, "cannot parse 'level' field")
} }
if in.Outlet == "" || in.Level == "" || in.Format == "" { formatter, err := parseLogFormat(common.Format)
err = errors.Errorf("must specify 'outlet', 'level' and 'format' field") if err != nil {
return return 0, nil, errors.Wrap(err, "cannot parse 'formatter' field")
}
return minLevel, formatter, nil
} }
minLevel, err := logger.ParseLevel(in.Level) var f EntryFormatter
if err != nil {
err = errors.Wrap(err, "cannot parse 'level' field")
return
}
formatter, err := parseLogFormat(in.Format)
if err != nil {
err = errors.Wrap(err, "cannot parse")
return
}
switch in.Outlet { switch v := in.Ret.(type) {
case "stdout": case config.StdoutLoggingOutlet:
o, err = parseStdoutOutlet(i, formatter) level, f, err = parseCommon(v.LoggingOutletCommon)
case "tcp": if err != nil {
o, err = parseTCPOutlet(i, formatter) break
case "syslog": }
o, err = parseSyslogOutlet(i, formatter) o, err = parseStdoutOutlet(v, f)
case config.TCPLoggingOutlet:
level, f, err = parseCommon(v.LoggingOutletCommon)
if err != nil {
break
}
o, err = parseTCPOutlet(v, f)
case config.SyslogLoggingOutlet:
level, f, err = parseCommon(v.LoggingOutletCommon)
if err != nil {
break
}
o, err = parseSyslogOutlet(v, f)
default: default:
err = errors.Errorf("unknown outlet type '%s'", in.Outlet) panic(v)
} }
return o, minLevel, err return o, level, err
} }
func parseStdoutOutlet(i interface{}, formatter EntryFormatter) (WriterOutlet, error) { func parseStdoutOutlet(in config.StdoutLoggingOutlet, formatter EntryFormatter) (WriterOutlet, error) {
var in struct {
Time bool
}
if err := mapstructure.Decode(i, &in); err != nil {
return WriterOutlet{}, errors.Wrap(err, "invalid structure for stdout outlet")
}
flags := MetadataAll flags := MetadataAll
writer := os.Stdout writer := os.Stdout
if !isatty.IsTerminal(writer.Fd()) && !in.Time { if !isatty.IsTerminal(writer.Fd()) && !in.Time {
@ -158,54 +149,22 @@ func parseStdoutOutlet(i interface{}, formatter EntryFormatter) (WriterOutlet, e
}, nil }, nil
} }
func parseTCPOutlet(i interface{}, formatter EntryFormatter) (out *TCPOutlet, err error) { func parseTCPOutlet(in config.TCPLoggingOutlet, formatter EntryFormatter) (out *TCPOutlet, err error) {
var in struct {
Net string
Address string
RetryInterval string `mapstructure:"retry_interval"`
TLS map[string]interface{}
}
if err = mapstructure.Decode(i, &in); err != nil {
return nil, errors.Wrap(err, "mapstructure error")
}
retryInterval, err := time.ParseDuration(in.RetryInterval)
if err != nil {
return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
}
if len(in.Net) == 0 {
return nil, errors.New("field 'net' must not be empty")
}
if len(in.Address) == 0 {
return nil, errors.New("field 'address' must not be empty")
}
var tlsConfig *tls.Config var tlsConfig *tls.Config
if in.TLS != nil { if in.TLS != nil {
tlsConfig, err = func(m map[string]interface{}, host string) (*tls.Config, error) { tlsConfig, err = func(m *config.TCPLoggingOutletTLS, host string) (*tls.Config, error) {
var in struct { clientCert, err := tls.LoadX509KeyPair(m.Cert, m.Key)
CA string
Cert string
Key string
}
if err := mapstructure.Decode(m, &in); err != nil {
return nil, errors.Wrap(err, "mapstructure error")
}
clientCert, err := tls.LoadX509KeyPair(in.Cert, in.Key)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "cannot load client cert") return nil, errors.Wrap(err, "cannot load client cert")
} }
var rootCAs *x509.CertPool var rootCAs *x509.CertPool
if in.CA == "" { if m.CA == "" {
if rootCAs, err = x509.SystemCertPool(); err != nil { if rootCAs, err = x509.SystemCertPool(); err != nil {
return nil, errors.Wrap(err, "cannot open system cert pool") return nil, errors.Wrap(err, "cannot open system cert pool")
} }
} else { } else {
rootCAs, err = tlsconf.ParseCAFile(in.CA) rootCAs, err = tlsconf.ParseCAFile(m.CA)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "cannot parse CA cert") return nil, errors.Wrap(err, "cannot parse CA cert")
} }
@ -222,30 +181,14 @@ func parseTCPOutlet(i interface{}, formatter EntryFormatter) (out *TCPOutlet, er
} }
formatter.SetMetadataFlags(MetadataAll) formatter.SetMetadataFlags(MetadataAll)
return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, retryInterval), nil return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, in.RetryInterval), nil
} }
func parseSyslogOutlet(i interface{}, formatter EntryFormatter) (out *SyslogOutlet, err error) { func parseSyslogOutlet(in config.SyslogLoggingOutlet, formatter EntryFormatter) (out *SyslogOutlet, err error) {
var in struct {
RetryInterval string `mapstructure:"retry_interval"`
}
if err = mapstructure.Decode(i, &in); err != nil {
return nil, errors.Wrap(err, "mapstructure error")
}
out = &SyslogOutlet{} out = &SyslogOutlet{}
out.Formatter = formatter out.Formatter = formatter
out.Formatter.SetMetadataFlags(MetadataNone) out.Formatter.SetMetadataFlags(MetadataNone)
out.RetryInterval = in.RetryInterval
out.RetryInterval = 0 // default to 0 as we assume local syslog will just work return out, nil
if in.RetryInterval != "" {
out.RetryInterval, err = time.ParseDuration(in.RetryInterval)
if err != nil {
return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
}
}
return
} }

14
cmd/daemon.go
View File

@ -4,14 +4,14 @@ import (
"context" "context"
"fmt" "fmt"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/zrepl/zrepl/cmd/config"
"github.com/zrepl/zrepl/logger"
"os" "os"
"os/signal" "os/signal"
"syscall" "syscall"
"time" "time"
"github.com/zrepl/zrepl/cmd/daemon" "github.com/zrepl/zrepl/cmd/daemon"
"github.com/zrepl/zrepl/cmd/daemon/job" "github.com/zrepl/zrepl/cmd/daemon/job"
"github.com/zrepl/zrepl/logger"
) )
// daemonCmd represents the daemon command // daemonCmd represents the daemon command
@ -75,13 +75,19 @@ func (a daemonJobAdaptor) Status() interface{} { return nil }
func doDaemon(cmd *cobra.Command, args []string) { func doDaemon(cmd *cobra.Command, args []string) {
conf, err := ParseConfig(rootArgs.configFile) conf, err := config.ParseConfig(rootArgs.configFile)
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "error parsing config: %s\n", err) fmt.Fprintf(os.Stderr, "error parsing config: %s\n", err)
os.Exit(1) os.Exit(1)
} }
log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second) outlets, err := parseLogging(conf.Global.Logging)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to generate logger: %s\n", err)
return
}
log := logger.NewLogger(outlets.Outlets, 1*time.Second)
ctx := WithLogger(context.Background(), log) ctx := WithLogger(context.Background(), log)
daemonJobs := make([]job.Job, 0, len(conf.Jobs)) daemonJobs := make([]job.Job, 0, len(conf.Jobs))