zrepl/cmd/config.go

package cmd

import (
	"errors"
	"fmt"
	"github.com/jinzhu/copier"
	"github.com/mitchellh/mapstructure"
	"github.com/zrepl/zrepl/jobrun"
	"github.com/zrepl/zrepl/rpc"
	"github.com/zrepl/zrepl/sshbytestream"
	. "github.com/zrepl/zrepl/util"
	"github.com/zrepl/zrepl/zfs"
	yaml "gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	JobSectionPush     string = "push"
	JobSectionPull     string = "pull"
	JobSectionPrune    string = "prune"
	JobSectionAutosnap string = "autosnap"
)

type Remote struct {
	Name      string
	Transport Transport
}

type Transport interface {
	Connect(rpcLog Logger) (rpc.RPCClient, error)
}
type LocalTransport struct {
}
type SSHTransport struct {
	Host                 string
	User                 string
	Port                 uint16
	IdentityFile         string   `mapstructure:"identity_file"`
	TransportOpenCommand []string `mapstructure:"transport_open_command"`
	SSHCommand           string   `mapstructure:"ssh_command"`
	Options              []string
	ConnLogReadFile      string `mapstructure:"connlog_read_file"`
	ConnLogWriteFile     string `mapstructure:"connlog_write_file"`
}

type Push struct {
	JobName           string // for use with jobrun package
	To                *Remote
	Filter            zfs.DatasetFilter
	InitialReplPolicy InitialReplPolicy
	RepeatStrategy    jobrun.RepeatStrategy
}
type Pull struct {
	JobName           string // for use with jobrun package
	From              *Remote
	Mapping           DatasetMapFilter
	InitialReplPolicy InitialReplPolicy
	RepeatStrategy    jobrun.RepeatStrategy
}

type Prune struct {
	JobName         string // for use with jobrun package
	DatasetFilter   zfs.DatasetFilter
	SnapshotFilter  zfs.FilesystemVersionFilter
	RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies?
}

type Autosnap struct {
	JobName       string // for use with jobrun package
	Prefix        string
	Interval      jobrun.RepeatStrategy
	DatasetFilter zfs.DatasetFilter
}

type Config struct {
	Remotes   map[string]*Remote
	Pushs     map[string]*Push            // job name -> job
	Pulls     map[string]*Pull            // job name -> job
	Sinks     map[string]DatasetMapFilter // client identity -> mapping
	PullACLs  map[string]DatasetMapFilter // client identity -> filter
	Prunes    map[string]*Prune           // job name -> job
	Autosnaps map[string]*Autosnap        // job name -> job
}

func ParseConfig(path string) (config Config, err error) {

	c := make(map[string]interface{}, 0)

	var bytes []byte

	if bytes, err = ioutil.ReadFile(path); err != nil {
		return
	}

	if err = yaml.Unmarshal(bytes, &c); err != nil {
		return
	}

	return parseMain(c)
}

func parseMain(root map[string]interface{}) (c Config, err error) {
	if c.Remotes, err = parseRemotes(root["remotes"]); err != nil {
		return
	}

	remoteLookup := func(name string) (remote *Remote, err error) {
		remote = c.Remotes[name]
		if remote == nil {
			err = fmt.Errorf("remote '%s' not defined", name)
		}
		return
	}

	if c.Pushs, err = parsePushs(root["pushs"], remoteLookup); err != nil {
		return
	}
	if c.Pulls, err = parsePulls(root["pulls"], remoteLookup); err != nil {
		return
	}
	if c.Sinks, err = parseSinks(root["sinks"]); err != nil {
		return
	}
	if c.PullACLs, err = parsePullACLs(root["pull_acls"]); err != nil {
		return
	}
	if c.Prunes, err = parsePrunes(root["prune"]); err != nil {
		return
	}
	if c.Autosnaps, err = parseAutosnaps(root["autosnap"]); err != nil {
		return
	}

	return
}

func fullJobName(section, name string) (full string, err error) {
	if len(name) < 1 {
		err = fmt.Errorf("job name not set")
		return
	}
	full = fmt.Sprintf("%s.%s", section, name)
	return
}

func parseRemotes(v interface{}) (remotes map[string]*Remote, err error) {

	asMap := make(map[string]struct {
		Transport map[string]interface{}
	}, 0)
	if err = mapstructure.Decode(v, &asMap); err != nil {
		return
	}

	remotes = make(map[string]*Remote, len(asMap))
	for name, p := range asMap {

		if name == LOCAL_TRANSPORT_IDENTITY {
			err = errors.New(fmt.Sprintf("remote name '%s' reserved for local pulls", LOCAL_TRANSPORT_IDENTITY))
			return
		}

		var transport Transport
		if transport, err = parseTransport(p.Transport); err != nil {
			return
		}
		remotes[name] = &Remote{
			Name:      name,
			Transport: transport,
		}
	}

	return
}

func parseTransport(it map[string]interface{}) (t Transport, err error) {

	if len(it) != 1 {
		err = errors.New("ambiguous transport type")
		return
	}

	for key, val := range it {
		switch key {
		case "ssh":
			t := SSHTransport{}
			if err = mapstructure.Decode(val, &t); err != nil {
				err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
				return nil, err
			}
			return t, nil
		default:
			return nil, errors.New(fmt.Sprintf("unknown transport type '%s'\n", key))
		}
	}

	return // unreachable

}

type remoteLookup func(name string) (*Remote, error)

func parsePushs(v interface{}, rl remoteLookup) (p map[string]*Push, err error) {

	asMap := make(map[string]struct {
		To                string
		Filter            map[string]string
		InitialReplPolicy string
		Repeat            map[string]string
	}, 0)

	if err = mapstructure.Decode(v, &asMap); err != nil {
		return
	}

	p = make(map[string]*Push, len(asMap))

	for name, e := range asMap {

		var toRemote *Remote
		if toRemote, err = rl(e.To); err != nil {
			return
		}
		push := &Push{
			To: toRemote,
		}

		if push.JobName, err = fullJobName(JobSectionPush, name); err != nil {
			return
		}
		if push.Filter, err = parseDatasetMapFilter(e.Filter, true); err != nil {
			return
		}

		if push.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
			return
		}

		if push.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
			return
		}

		p[name] = push
	}

	return
}

func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error) {

	asMap := make(map[string]struct {
		From              string
		Mapping           map[string]string
		InitialReplPolicy string
		Repeat            map[string]string
	}, 0)

	if err = mapstructure.Decode(v, &asMap); err != nil {
		return
	}

	p = make(map[string]*Pull, len(asMap))

	for name, e := range asMap {

		if len(e.From) < 1 {
			err = fmt.Errorf("source not set ('from' attribute is empty)")
			return
		}

		var fromRemote *Remote

		if e.From == LOCAL_TRANSPORT_IDENTITY {
			fromRemote = &Remote{
				Name:      LOCAL_TRANSPORT_IDENTITY,
				Transport: LocalTransport{},
			}
		} else {
			if fromRemote, err = rl(e.From); err != nil {
				return
			}
		}

		pull := &Pull{
			From: fromRemote,
		}
		if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil {
			return
		}
		if pull.Mapping, err = parseDatasetMapFilter(e.Mapping, false); err != nil {
			return
		}
		if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
			return
		}
		if pull.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
			return
		}

		p[name] = pull
	}

	return
}

func parseInitialReplPolicy(v interface{}, defaultPolicy InitialReplPolicy) (p InitialReplPolicy, err error) {
	s, ok := v.(string)
	if !ok {
		goto err
	}

	switch {
	case s == "":
		p = defaultPolicy
	case s == "most_recent":
		p = InitialReplPolicyMostRecent
	case s == "all":
		p = InitialReplPolicyAll
	default:
		goto err
	}

	return

err:
	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
	return
}

func parseRepeatStrategy(r map[string]string) (s jobrun.RepeatStrategy, err error) {

	if r == nil {
		return jobrun.NoRepeatStrategy{}, nil
	}

	if repeatStr, ok := r["interval"]; ok {
		d, err := parseDuration(repeatStr)
		if err != nil {
			return nil, err
		}
		s = &jobrun.PeriodicRepeatStrategy{d}
		return s, err
	} else {
		return nil, fmt.Errorf("attribute 'interval' not found but required in repeat specification")
	}

}

func expectList(v interface{}) (asList []interface{}, err error) {
	var ok bool
	if asList, ok = v.([]interface{}); !ok {
		err = errors.New("expected list")
	}
	return
}

func parseSinks(v interface{}) (m map[string]DatasetMapFilter, err error) {
	var asMap map[string]map[string]interface{}
	if err = mapstructure.Decode(v, &asMap); err != nil {
		return
	}

	m = make(map[string]DatasetMapFilter, len(asMap))

	for identity, entry := range asMap {
		parseSink := func() (mapping DatasetMapFilter, err error) {
			mappingMap, ok := entry["mapping"]
			if !ok {
				err = fmt.Errorf("no mapping specified")
				return
			}
			mapping, err = parseDatasetMapFilter(mappingMap, false)
			return
		}
		mapping, sinkErr := parseSink()
		if sinkErr != nil {
			err = fmt.Errorf("cannot parse sink for identity '%s': %s", identity, sinkErr)
			return
		}
		m[identity] = mapping
	}
	return
}

func parsePullACLs(v interface{}) (m map[string]DatasetMapFilter, err error) {
	var asMap map[string]map[string]interface{}
	if err = mapstructure.Decode(v, &asMap); err != nil {
		return
	}

	m = make(map[string]DatasetMapFilter, len(asMap))

	for identity, entry := range asMap {
		parsePullACL := func() (filter DatasetMapFilter, err error) {
			filterMap, ok := entry["filter"]
			if !ok {
				err = fmt.Errorf("no filter specified")
				return
			}
			filter, err = parseDatasetMapFilter(filterMap, true)
			return
		}
		filter, filterErr := parsePullACL()
		if filterErr != nil {
			err = fmt.Errorf("cannot parse pull-ACL for identity '%s': %s", identity, filterErr)
			return
		}
		m[identity] = filter
	}
	return
}

func parseDatasetMapFilter(mi interface{}, filterOnly bool) (f DatasetMapFilter, err error) {

	var m map[string]string
	if err = mapstructure.Decode(mi, &m); err != nil {
		err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
		return
	}

	f = NewDatasetMapFilter(len(m), filterOnly)
	for pathPattern, mapping := range m {
		if err = f.Add(pathPattern, mapping); err != nil {
			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
			return
		}
	}
	return
}

func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCClient, err error) {
	var stream io.ReadWriteCloser
	var rpcTransport sshbytestream.SSHTransport
	if err = copier.Copy(&rpcTransport, t); err != nil {
		return
	}
	if stream, err = sshbytestream.Outgoing(rpcTransport); err != nil {
		return
	}
	stream, err = NewReadWriteCloserLogger(stream, t.ConnLogReadFile, t.ConnLogWriteFile)
	if err != nil {
		return
	}
	client := rpc.NewClient(stream)
	return client, nil
}

func (t LocalTransport) Connect(rpcLog Logger) (r rpc.RPCClient, err error) {
	local := rpc.NewLocalRPC()
	handler := Handler{
		Logger: log,
		// Allow access to any dataset since we control what mapping
		// is passed to the pull routine.
		// All local datasets will be passed to its Map() function,
		// but only those for which a mapping exists will actually be pulled.
		// We can pay this small performance penalty for now.
		PullACL: localPullACL{},
	}
	registerEndpoints(local, handler)
	return local, nil
}

func parsePrunes(m interface{}) (rets map[string]*Prune, err error) {

	asList := make(map[string]map[string]interface{}, 0)
	if err = mapstructure.Decode(m, &asList); err != nil {
		return
	}

	rets = make(map[string]*Prune, len(asList))

	for name, e := range asList {
		var prune *Prune
		if prune, err = parsePrune(e, name); err != nil {
			err = fmt.Errorf("cannot parse prune job %s: %s", name, err)
			return
		}
		rets[name] = prune
	}

	return
}

func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error) {

	// Only support grid policy for now
	policyName, ok := e["policy"]
	if !ok || policyName != "grid" {
		err = fmt.Errorf("prune job with unimplemented policy '%s'", policyName)
		return
	}

	var i struct {
		Grid           string
		DatasetFilter  map[string]string `mapstructure:"dataset_filter"`
		SnapshotFilter map[string]string `mapstructure:"snapshot_filter"`
	}

	if err = mapstructure.Decode(e, &i); err != nil {
		return
	}

	prune = &Prune{}

	if prune.JobName, err = fullJobName(JobSectionPrune, name); err != nil {
		return
	}

	// Parse grid policy
	intervals, err := parseRetentionGridIntervalsString(i.Grid)
	if err != nil {
		err = fmt.Errorf("cannot parse retention grid: %s", err)
		return
	}
	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
	lastDuration := time.Duration(0)
	for i := range intervals {
		if intervals[i].Length < lastDuration {
			err = fmt.Errorf("retention grid interval length must be monotonically increasing:"+
				"interval %d is shorter than %d", i+1, i)
			return
		} else {
			lastDuration = intervals[i].Length
		}
	}
	prune.RetentionPolicy = NewRetentionGrid(intervals)

	// Parse filters
	if prune.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
		err = fmt.Errorf("cannot parse dataset filter: %s", err)
		return
	}
	if prune.SnapshotFilter, err = parseSnapshotFilter(i.SnapshotFilter); err != nil {
		err = fmt.Errorf("cannot parse snapshot filter: %s", err)
		return
	}

	return
}

var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)

func parseDuration(e string) (d time.Duration, err error) {
	comps := durationStringRegex.FindStringSubmatch(e)
	if len(comps) != 3 {
		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
		return
	}

	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
	if err != nil {
		return
	}

	var durationUnit time.Duration
	switch comps[2] {
	case "s":
		durationUnit = time.Second
	case "m":
		durationUnit = time.Minute
	case "h":
		durationUnit = time.Hour
	case "d":
		durationUnit = 24 * time.Hour
	case "w":
		durationUnit = 24 * 7 * time.Hour
	default:
		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
		return
	}

	d = time.Duration(durationFactor) * durationUnit
	return

}

var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)

func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {

	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
	if comps == nil {
		err = fmt.Errorf("retention string does not match expected format")
		return
	}

	times, err := strconv.Atoi(comps[1])
	if err != nil {
		return nil, err
	} else if times <= 0 {
		return nil, fmt.Errorf("contains factor <= 0")
	}

	duration, err := parseDuration(comps[2])
	if err != nil {
		return nil, err
	}

	keepCount := 1
	if comps[3] != "" {
		// Decompose key=value, comma separated
		// For now, only keep_count is supported
		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
		res := re.FindStringSubmatch(comps[4])
		if res == nil || len(res) != 2 {
			err = fmt.Errorf("interval parameter contains unknown parameters")
			return
		}
		if res[1] == "all" {
			keepCount = RetentionGridKeepCountAll
		} else {
			keepCount, err = strconv.Atoi(res[1])
			if err != nil {
				err = fmt.Errorf("cannot parse keep_count value")
				return
			}
		}
	}

	intervals = make([]RetentionInterval, times)
	for i := range intervals {
		intervals[i] = RetentionInterval{
			Length:    duration,
			KeepCount: keepCount,
		}
	}

	return

}

func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {

	ges := strings.Split(s, "|")
	intervals = make([]RetentionInterval, 0, 7*len(ges))

	for intervalIdx, e := range ges {
		parsed, err := parseRetentionGridIntervalString(e)
		if err != nil {
			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
		}
		intervals = append(intervals, parsed...)
	}

	return
}

type prefixSnapshotFilter struct {
	prefix string
}

func (f prefixSnapshotFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
	return fsv.Type == zfs.Snapshot && strings.HasPrefix(fsv.Name, f.prefix), nil
}

func parseSnapshotFilter(fm map[string]string) (snapFilter zfs.FilesystemVersionFilter, err error) {
	prefix, ok := fm["prefix"]
	if !ok {
		err = fmt.Errorf("unsupported snapshot filter")
		return
	}
	snapFilter = prefixSnapshotFilter{prefix}
	return
}

func parseAutosnaps(m interface{}) (snaps map[string]*Autosnap, err error) {

	asMap := make(map[string]interface{}, 0)
	if err = mapstructure.Decode(m, &asMap); err != nil {
		return
	}

	snaps = make(map[string]*Autosnap, len(asMap))

	for name, e := range asMap {
		var snap *Autosnap
		if snap, err = parseAutosnap(e, name); err != nil {
			err = fmt.Errorf("cannot parse autonsap job %s: %s", name, err)
			return
		}
		snaps[name] = snap
	}

	return

}

func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) {

	var i struct {
		Prefix        string
		Interval      string
		DatasetFilter map[string]string `mapstructure:"dataset_filter"`
	}

	if err = mapstructure.Decode(m, &i); err != nil {
		err = fmt.Errorf("structure unfit: %s", err)
		return
	}

	a = &Autosnap{}

	if a.JobName, err = fullJobName(JobSectionAutosnap, name); err != nil {
		return
	}

	if len(i.Prefix) < 1 {
		err = fmt.Errorf("prefix must not be empty")
		return
	}
	a.Prefix = i.Prefix

	var interval time.Duration
	if interval, err = parseDuration(i.Interval); err != nil {
		err = fmt.Errorf("cannot parse interval: %s", err)
		return
	}
	a.Interval = &jobrun.PeriodicRepeatStrategy{interval}

	if len(i.DatasetFilter) == 0 {
		err = fmt.Errorf("dataset_filter not specified")
		return
	}
	if a.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
		err = fmt.Errorf("cannot parse dataset filter: %s", err)
	}

	return
}