2017-09-10 16:13:05 +02:00
|
|
|
package cmd
|
|
|
|
|
|
|
|
import (
|
|
|
|
"io/ioutil"
|
|
|
|
|
2017-09-11 13:43:18 +02:00
|
|
|
"fmt"
|
2018-08-25 22:23:47 +02:00
|
|
|
"github.com/go-yaml/yaml"
|
2017-09-10 16:13:05 +02:00
|
|
|
"github.com/mitchellh/mapstructure"
|
|
|
|
"github.com/pkg/errors"
|
2018-08-25 21:30:25 +02:00
|
|
|
"github.com/problame/go-streamrpc"
|
2017-09-17 18:20:05 +02:00
|
|
|
"os"
|
2017-10-05 15:12:50 +02:00
|
|
|
"regexp"
|
|
|
|
"strconv"
|
|
|
|
"time"
|
2017-09-10 16:13:05 +02:00
|
|
|
)
|
|
|
|
|
2017-09-17 18:20:05 +02:00
|
|
|
var ConfigFileDefaultLocations []string = []string{
|
|
|
|
"/etc/zrepl/zrepl.yml",
|
|
|
|
"/usr/local/etc/zrepl/zrepl.yml",
|
|
|
|
}
|
|
|
|
|
2017-09-17 23:54:23 +02:00
|
|
|
const (
|
|
|
|
JobNameControl string = "control"
|
|
|
|
)
|
|
|
|
|
|
|
|
var ReservedJobNames []string = []string{
|
|
|
|
JobNameControl,
|
|
|
|
}
|
|
|
|
|
2017-09-17 18:20:05 +02:00
|
|
|
type ConfigParsingContext struct {
|
|
|
|
Global *Global
|
|
|
|
}
|
|
|
|
|
2017-09-22 14:02:07 +02:00
|
|
|
func ParseConfig(path string) (config *Config, err error) {
|
2017-09-17 18:20:05 +02:00
|
|
|
|
|
|
|
if path == "" {
|
|
|
|
// Try default locations
|
|
|
|
for _, l := range ConfigFileDefaultLocations {
|
|
|
|
stat, err := os.Stat(l)
|
|
|
|
if err != nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if !stat.Mode().IsRegular() {
|
2017-09-22 14:02:07 +02:00
|
|
|
err = errors.Errorf("file at default location is not a regular file: %s", l)
|
2017-09-17 18:20:05 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
path = l
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
2017-09-10 16:13:05 +02:00
|
|
|
|
|
|
|
var i interface{}
|
|
|
|
|
|
|
|
var bytes []byte
|
|
|
|
|
|
|
|
if bytes, err = ioutil.ReadFile(path); err != nil {
|
|
|
|
err = errors.WithStack(err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if err = yaml.Unmarshal(bytes, &i); err != nil {
|
|
|
|
err = errors.WithStack(err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
return parseConfig(i)
|
|
|
|
}
|
|
|
|
|
|
|
|
func parseConfig(i interface{}) (c *Config, err error) {
|
|
|
|
|
2017-09-11 13:43:18 +02:00
|
|
|
var asMap struct {
|
|
|
|
Global map[string]interface{}
|
|
|
|
Jobs []map[string]interface{}
|
2017-09-10 16:13:05 +02:00
|
|
|
}
|
2017-09-11 13:43:18 +02:00
|
|
|
if err := mapstructure.Decode(i, &asMap); err != nil {
|
|
|
|
return nil, errors.Wrap(err, "config root must be a dict")
|
2017-09-10 16:13:05 +02:00
|
|
|
}
|
|
|
|
|
2017-09-11 13:43:18 +02:00
|
|
|
c = &Config{}
|
2017-09-10 16:13:05 +02:00
|
|
|
|
2017-09-11 13:43:18 +02:00
|
|
|
// Parse global with defaults
|
|
|
|
c.Global.Serve.Stdinserver.SockDir = "/var/run/zrepl/stdinserver"
|
2017-09-17 23:54:23 +02:00
|
|
|
c.Global.Control.Sockpath = "/var/run/zrepl/control"
|
|
|
|
|
2017-09-11 13:43:18 +02:00
|
|
|
err = mapstructure.Decode(asMap.Global, &c.Global)
|
|
|
|
if err != nil {
|
2017-09-22 14:13:58 +02:00
|
|
|
err = errors.Wrap(err, "mapstructure error on 'global' section: %s")
|
2017-09-11 13:43:18 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-09-22 14:13:58 +02:00
|
|
|
if c.Global.logging, err = parseLogging(asMap.Global["logging"]); err != nil {
|
|
|
|
return nil, errors.Wrap(err, "cannot parse logging section")
|
|
|
|
}
|
|
|
|
|
2017-09-17 18:20:05 +02:00
|
|
|
cpc := ConfigParsingContext{&c.Global}
|
|
|
|
jpc := JobParsingContext{cpc}
|
2017-09-11 13:43:18 +02:00
|
|
|
c.Jobs = make(map[string]Job, len(asMap.Jobs))
|
2018-04-14 11:24:47 +02:00
|
|
|
|
|
|
|
// FIXME internal jobs should not be mixed with user jobs
|
|
|
|
// Monitoring Jobs
|
|
|
|
var monJobs []map[string]interface{}
|
|
|
|
if err := mapstructure.Decode(asMap.Global["monitoring"], &monJobs); err != nil {
|
|
|
|
return nil, errors.Wrap(err, "cannot parse monitoring section")
|
|
|
|
}
|
|
|
|
for i, jc := range monJobs {
|
|
|
|
if jc["name"] == "" || jc["name"] == nil {
|
|
|
|
// FIXME internal jobs should not require a name...
|
|
|
|
jc["name"] = fmt.Sprintf("prometheus-%d", i)
|
|
|
|
}
|
|
|
|
job, err := parseJob(jpc, jc)
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrapf(err, "cannot parse monitoring job #%d", i)
|
|
|
|
}
|
|
|
|
if job.JobType() != JobTypePrometheus {
|
|
|
|
return nil, errors.Errorf("monitoring job #%d has invalid job type", i)
|
|
|
|
}
|
|
|
|
c.Jobs[job.JobName()] = job
|
|
|
|
}
|
|
|
|
|
|
|
|
// Regular Jobs
|
2017-09-11 13:43:18 +02:00
|
|
|
for i := range asMap.Jobs {
|
2017-09-17 18:20:05 +02:00
|
|
|
job, err := parseJob(jpc, asMap.Jobs[i])
|
2017-09-10 16:13:05 +02:00
|
|
|
if err != nil {
|
2017-09-11 13:43:18 +02:00
|
|
|
// Try to find its name
|
|
|
|
namei, ok := asMap.Jobs[i]["name"]
|
|
|
|
if !ok {
|
2017-09-30 16:31:55 +02:00
|
|
|
namei = fmt.Sprintf("<no name, entry #%d in list>", i)
|
2017-09-11 13:43:18 +02:00
|
|
|
}
|
|
|
|
err = errors.Wrapf(err, "cannot parse job '%v'", namei)
|
2017-09-10 16:13:05 +02:00
|
|
|
return nil, err
|
|
|
|
}
|
2017-09-17 23:54:23 +02:00
|
|
|
jn := job.JobName()
|
|
|
|
if _, ok := c.Jobs[jn]; ok {
|
2018-04-14 11:24:47 +02:00
|
|
|
err = errors.Errorf("duplicate or invalid job name: %s", jn)
|
2017-09-17 23:54:23 +02:00
|
|
|
return nil, err
|
|
|
|
}
|
2017-09-11 13:43:18 +02:00
|
|
|
c.Jobs[job.JobName()] = job
|
2017-09-10 16:13:05 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return c, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2017-09-11 13:43:18 +02:00
|
|
|
func extractStringField(i map[string]interface{}, key string, notempty bool) (field string, err error) {
|
|
|
|
vi, ok := i[key]
|
2017-09-10 16:13:05 +02:00
|
|
|
if !ok {
|
2017-09-11 13:43:18 +02:00
|
|
|
err = errors.Errorf("must have field '%s'", key)
|
|
|
|
return "", err
|
2017-09-10 16:13:05 +02:00
|
|
|
}
|
2017-09-11 13:43:18 +02:00
|
|
|
field, ok = vi.(string)
|
2017-09-10 16:13:05 +02:00
|
|
|
if !ok {
|
2017-09-11 13:43:18 +02:00
|
|
|
err = errors.Errorf("'%s' field must have type string", key)
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
if notempty && len(field) <= 0 {
|
|
|
|
err = errors.Errorf("'%s' field must not be empty", key)
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-09-17 18:20:05 +02:00
|
|
|
type JobParsingContext struct {
|
|
|
|
ConfigParsingContext
|
|
|
|
}
|
|
|
|
|
|
|
|
func parseJob(c JobParsingContext, i map[string]interface{}) (j Job, err error) {
|
2017-09-11 13:43:18 +02:00
|
|
|
|
|
|
|
name, err := extractStringField(i, "name", true)
|
|
|
|
if err != nil {
|
2018-04-14 11:25:12 +02:00
|
|
|
return nil, err
|
2017-09-10 16:13:05 +02:00
|
|
|
}
|
|
|
|
|
2017-09-17 23:54:23 +02:00
|
|
|
for _, r := range ReservedJobNames {
|
|
|
|
if name == r {
|
|
|
|
err = errors.Errorf("job name '%s' is reserved", name)
|
2018-04-14 11:25:12 +02:00
|
|
|
return nil, err
|
2017-09-17 23:54:23 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-05 22:22:55 +02:00
|
|
|
jobtypeStr, err := extractStringField(i, "type", true)
|
|
|
|
if err != nil {
|
2018-04-14 11:25:12 +02:00
|
|
|
return nil, err
|
2018-04-05 22:22:55 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
jobtype, err := ParseUserJobType(jobtypeStr)
|
2017-09-11 13:43:18 +02:00
|
|
|
if err != nil {
|
2018-04-14 11:25:12 +02:00
|
|
|
return nil, err
|
2017-09-11 13:43:18 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
switch jobtype {
|
2018-04-05 22:22:55 +02:00
|
|
|
case JobTypePull:
|
2017-09-17 18:20:05 +02:00
|
|
|
return parsePullJob(c, name, i)
|
2018-04-05 22:22:55 +02:00
|
|
|
case JobTypeSource:
|
2017-09-17 18:20:05 +02:00
|
|
|
return parseSourceJob(c, name, i)
|
2018-04-05 22:22:55 +02:00
|
|
|
case JobTypeLocal:
|
2017-09-17 18:20:05 +02:00
|
|
|
return parseLocalJob(c, name, i)
|
2017-09-10 16:13:05 +02:00
|
|
|
default:
|
2018-04-05 22:22:55 +02:00
|
|
|
panic(fmt.Sprintf("implementation error: unknown job type %s", jobtype))
|
2017-09-10 16:13:05 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2018-08-08 13:09:51 +02:00
|
|
|
func parseConnect(i map[string]interface{}) (c streamrpc.Connecter, err error) {
|
2017-09-11 13:43:18 +02:00
|
|
|
|
|
|
|
t, err := extractStringField(i, "type", true)
|
|
|
|
if err != nil {
|
2017-09-10 16:13:05 +02:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-09-11 13:43:18 +02:00
|
|
|
switch t {
|
2017-09-10 16:13:05 +02:00
|
|
|
case "ssh+stdinserver":
|
|
|
|
return parseSSHStdinserverConnecter(i)
|
2018-08-25 12:58:17 +02:00
|
|
|
case "tcp":
|
|
|
|
return parseTCPConnecter(i)
|
2017-09-10 16:13:05 +02:00
|
|
|
default:
|
2017-09-11 13:43:18 +02:00
|
|
|
return nil, errors.Errorf("unknown connection type '%s'", t)
|
2017-09-10 16:13:05 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2018-02-17 20:48:31 +01:00
|
|
|
func parsePrunePolicy(v map[string]interface{}, willSeeBookmarks bool) (p PrunePolicy, err error) {
|
2017-09-10 16:13:05 +02:00
|
|
|
|
2017-09-11 13:43:18 +02:00
|
|
|
policyName, err := extractStringField(v, "policy", true)
|
|
|
|
if err != nil {
|
2017-09-10 16:13:05 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
switch policyName {
|
|
|
|
case "grid":
|
2018-02-17 20:48:31 +01:00
|
|
|
return parseGridPrunePolicy(v, willSeeBookmarks)
|
2017-09-13 23:46:15 +02:00
|
|
|
case "noprune":
|
|
|
|
return NoPrunePolicy{}, nil
|
2017-09-10 16:13:05 +02:00
|
|
|
default:
|
|
|
|
err = errors.Errorf("unknown policy '%s'", policyName)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-08 13:09:51 +02:00
|
|
|
func parseAuthenticatedChannelListenerFactory(c JobParsingContext, v map[string]interface{}) (p ListenerFactory, err error) {
|
2017-09-10 16:13:05 +02:00
|
|
|
|
2017-09-11 13:43:18 +02:00
|
|
|
t, err := extractStringField(v, "type", true)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2017-09-10 16:13:05 +02:00
|
|
|
}
|
|
|
|
|
2017-09-11 13:43:18 +02:00
|
|
|
switch t {
|
2017-09-10 16:13:05 +02:00
|
|
|
case "stdinserver":
|
2017-09-17 18:20:05 +02:00
|
|
|
return parseStdinserverListenerFactory(c, v)
|
2018-08-25 12:58:17 +02:00
|
|
|
case "tcp":
|
|
|
|
return parseTCPListenerFactory(c, v)
|
2017-09-10 16:13:05 +02:00
|
|
|
default:
|
2017-09-11 13:43:18 +02:00
|
|
|
err = errors.Errorf("unknown type '%s'", t)
|
2017-09-10 16:13:05 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
2017-10-05 15:12:50 +02:00
|
|
|
|
|
|
|
var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
|
|
|
|
|
|
|
|
func parsePostitiveDuration(e string) (d time.Duration, err error) {
|
|
|
|
comps := durationStringRegex.FindStringSubmatch(e)
|
|
|
|
if len(comps) != 3 {
|
|
|
|
err = fmt.Errorf("does not match regex: %s %#v", e, comps)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
if durationFactor <= 0 {
|
|
|
|
return 0, errors.New("duration must be positive integer")
|
|
|
|
}
|
|
|
|
|
|
|
|
var durationUnit time.Duration
|
|
|
|
switch comps[2] {
|
|
|
|
case "s":
|
|
|
|
durationUnit = time.Second
|
|
|
|
case "m":
|
|
|
|
durationUnit = time.Minute
|
|
|
|
case "h":
|
|
|
|
durationUnit = time.Hour
|
|
|
|
case "d":
|
|
|
|
durationUnit = 24 * time.Hour
|
|
|
|
case "w":
|
|
|
|
durationUnit = 24 * 7 * time.Hour
|
|
|
|
default:
|
|
|
|
err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
d = time.Duration(durationFactor) * durationUnit
|
|
|
|
return
|
|
|
|
}
|