extern/zrepl
1
0
Fork 0
mirror of https://github.com/zrepl/zrepl.git synced 2025-02-16 10:29:54 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Simplify CLI by requiring explicit job names.

Browse Source 
Job names are derived from job type + user-defined name in config file
CLI now has subcommands corresponding 1:1 to the config file sections:

        push,pull,autosnap,prune

A subcommand always expects a job name, thus executes exactly one job.

Dict-style syntax also used for PullACL and Sink sections.

jobrun package is currently only used for autosnap, all others need to
be invoked repeatedly via external tool.
Plan is to re-integrate jobrun in an explicit daemon-mode (subcommand).
This commit is contained in:
Christian Schwarz 2017-07-06 15:36:53 +02:00
parent b44a005bbb
commit e951beaef5
6 changed files with 234 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
cmd/autosnap.go
View File

@ -5,14 +5,11 @@ import (
"github.com/spf13/cobra"
"github.com/zrepl/zrepl/jobrun"
"github.com/zrepl/zrepl/zfs"
"os"
"sync"
"time"
)
var autosnapArgs struct {
job string
}
var AutosnapCmd = &cobra.Command{
Use: "autosnap",
Short: "perform automatic snapshotting",
@ -20,7 +17,6 @@ var AutosnapCmd = &cobra.Command{
}
func init() {
AutosnapCmd.Flags().StringVar(&autosnapArgs.job, "job", "", "job to run")
RootCmd.AddCommand(AutosnapCmd)
}
@ -33,34 +29,34 @@ func cmdAutosnap(cmd *cobra.Command, args []string) {
runner.Start()
}()
log.Printf("autosnap...")
for i := range conf.Autosnaps {
snap := conf.Autosnaps[i]
if autosnapArgs.job == "" || autosnapArgs.job == snap.Name {
job := jobrun.Job{
Name: fmt.Sprintf("autosnap.%s", snap.Name),
RepeatStrategy: snap.Interval,
RunFunc: func(log jobrun.Logger) error {
log.Printf("doing autosnap: %v", snap)
ctx := AutosnapContext{snap}
return doAutosnap(ctx, log)
},
}
runner.AddJob(job)
}
if len(args) < 1 {
log.Printf("must specify exactly one job as positional argument")
os.Exit(1)
}
snap, ok := conf.Autosnaps[args[0]]
if !ok {
log.Printf("could not find autosnap job: %s", args[0])
os.Exit(1)
}
job := jobrun.Job{
Name: snap.JobName,
RepeatStrategy: snap.Interval,
RunFunc: func(log jobrun.Logger) error {
log.Printf("doing autosnap: %v", snap)
ctx := AutosnapContext{snap}
return doAutosnap(ctx, log)
},
}
runner.AddJob(job)
wg.Wait()
}
type AutosnapContext struct {
Autosnap Autosnap
Autosnap *Autosnap
}
func doAutosnap(ctx AutosnapContext, log Logger) (err error) {

179
cmd/config.go
View File

@ -19,6 +19,13 @@ import (
"time"
)
var (
JobSectionPush string = "push"
JobSectionPull string = "pull"
JobSectionPrune string = "prune"
JobSectionAutosnap string = "autosnap"
)
type Pool struct {
Name string
Transport Transport
@ -43,12 +50,14 @@ type SSHTransport struct {
}
type Push struct {
JobName string // for use with jobrun package
To *Pool
Filter zfs.DatasetMapping
InitialReplPolicy rpc.InitialReplPolicy
RepeatStrategy jobrun.RepeatStrategy
}
type Pull struct {
JobName string // for use with jobrun package
From *Pool
Mapping zfs.DatasetMapping
InitialReplPolicy rpc.InitialReplPolicy
@ -61,27 +70,27 @@ type ClientMapping struct {
}
type Prune struct {
Name string
JobName string // for use with jobrun package
DatasetFilter zfs.DatasetMapping
SnapshotFilter zfs.FilesystemVersionFilter
RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies?
}
type Autosnap struct {
Name string
JobName string // for use with jobrun package
Prefix string
Interval jobrun.RepeatStrategy
DatasetFilter zfs.DatasetMapping
}
type Config struct {
Pools []Pool
Pushs []Push
Pulls []Pull
Sinks []ClientMapping
PullACLs []ClientMapping
Prunes []Prune
Autosnaps []Autosnap
Pools map[string]*Pool
Pushs map[string]*Push // job name -> job
Pulls map[string]*Pull // job name -> job
Sinks map[string]*ClientMapping // client identity -> mapping
PullACLs map[string]*ClientMapping // client identity -> mapping
Prunes map[string]*Prune // job name -> job
Autosnaps map[string]*Autosnap // job name -> job
}
func ParseConfig(path string) (config Config, err error) {
@ -106,13 +115,12 @@ func parseMain(root map[string]interface{}) (c Config, err error) {
return
}
poolLookup := func(name string) (*Pool, error) {
for _, pool := range c.Pools {
if pool.Name == name {
return &pool, nil
}
poolLookup := func(name string) (pool *Pool, err error) {
pool = c.Pools[name]
if pool == nil {
err = fmt.Errorf("pool '%s' not defined", name)
}
return nil, errors.New(fmt.Sprintf("pool '%s' not defined", name))
return
}
if c.Pushs, err = parsePushs(root["pushs"], poolLookup); err != nil {
@ -133,23 +141,32 @@ func parseMain(root map[string]interface{}) (c Config, err error) {
if c.Autosnaps, err = parseAutosnaps(root["autosnap"]); err != nil {
return
}
return
}
func parsePools(v interface{}) (pools []Pool, err error) {
func fullJobName(section, name string) (full string, err error) {
if len(name) < 1 {
err = fmt.Errorf("job name not set")
return
}
full = fmt.Sprintf("%s.%s", section, name)
return
}
asList := make([]struct {
Name string
func parsePools(v interface{}) (pools map[string]*Pool, err error) {
asMap := make(map[string]struct {
Transport map[string]interface{}
}, 0)
if err = mapstructure.Decode(v, &asList); err != nil {
if err = mapstructure.Decode(v, &asMap); err != nil {
return
}
pools = make([]Pool, len(asList))
for i, p := range asList {
pools = make(map[string]*Pool, len(asMap))
for name, p := range asMap {
if p.Name == rpc.LOCAL_TRANSPORT_IDENTITY {
if name == rpc.LOCAL_TRANSPORT_IDENTITY {
err = errors.New(fmt.Sprintf("pool name '%s' reserved for local pulls", rpc.LOCAL_TRANSPORT_IDENTITY))
return
}
@ -158,8 +175,8 @@ func parsePools(v interface{}) (pools []Pool, err error) {
if transport, err = parseTransport(p.Transport); err != nil {
return
}
pools[i] = Pool{
Name: p.Name,
pools[name] = &Pool{
Name: name,
Transport: transport,
}
}
@ -194,31 +211,34 @@ func parseTransport(it map[string]interface{}) (t Transport, err error) {
type poolLookup func(name string) (*Pool, error)
func parsePushs(v interface{}, pl poolLookup) (p []Push, err error) {
func parsePushs(v interface{}, pl poolLookup) (p map[string]*Push, err error) {
asList := make([]struct {
asMap := make(map[string]struct {
To string
Filter map[string]string
InitialReplPolicy string
Repeat map[string]string
}, 0)
if err = mapstructure.Decode(v, &asList); err != nil {
if err = mapstructure.Decode(v, &asMap); err != nil {
return
}
p = make([]Push, len(asList))
p = make(map[string]*Push, len(asMap))
for i, e := range asList {
for name, e := range asMap {
var toPool *Pool
if toPool, err = pl(e.To); err != nil {
return
}
push := Push{
push := &Push{
To: toPool,
}
if push.JobName, err = fullJobName(JobSectionPush, name); err != nil {
return
}
if push.Filter, err = parseComboMapping(e.Filter); err != nil {
return
}
@ -231,34 +251,39 @@ func parsePushs(v interface{}, pl poolLookup) (p []Push, err error) {
return
}
p[i] = push
p[name] = push
}
return
}
func parsePulls(v interface{}, pl poolLookup) (p []Pull, err error) {
func parsePulls(v interface{}, pl poolLookup) (p map[string]*Pull, err error) {
asList := make([]struct {
asMap := make(map[string]struct {
From string
Mapping map[string]string
InitialReplPolicy string
Repeat map[string]string
}, 0)
if err = mapstructure.Decode(v, &asList); err != nil {
if err = mapstructure.Decode(v, &asMap); err != nil {
return
}
p = make([]Pull, len(asList))
p = make(map[string]*Pull, len(asMap))
for i, e := range asList {
for name, e := range asMap {
if len(e.From) < 1 {
err = fmt.Errorf("source pool not set (from attribute is empty)")
return
}
var fromPool *Pool
if e.From == rpc.LOCAL_TRANSPORT_IDENTITY {
fromPool = &Pool{
Name: "local",
Name: rpc.LOCAL_TRANSPORT_IDENTITY,
Transport: LocalTransport{},
}
} else {
@ -267,9 +292,12 @@ func parsePulls(v interface{}, pl poolLookup) (p []Pull, err error) {
}
}
pull := Pull{
pull := &Pull{
From: fromPool,
}
if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil {
return
}
if pull.Mapping, err = parseComboMapping(e.Mapping); err != nil {
return
}
@ -280,7 +308,7 @@ func parsePulls(v interface{}, pl poolLookup) (p []Pull, err error) {
return
}
p[i] = pull
p[name] = pull
}
return
@ -337,35 +365,35 @@ func expectList(v interface{}) (asList []interface{}, err error) {
return
}
func parseClientMappings(v interface{}) (cm []ClientMapping, err error) {
func parseClientMappings(v interface{}) (cm map[string]*ClientMapping, err error) {
var asList []interface{}
if asList, err = expectList(v); err != nil {
var asMap map[string]interface{}
if err = mapstructure.Decode(v, &asMap); err != nil {
return
}
cm = make([]ClientMapping, len(asList))
cm = make(map[string]*ClientMapping, len(asMap))
for i, e := range asList {
var m ClientMapping
if m, err = parseClientMapping(e); err != nil {
for identity, e := range asMap {
var m *ClientMapping
if m, err = parseClientMapping(e, identity); err != nil {
return
}
cm[i] = m
cm[identity] = m
}
return
}
func parseClientMapping(v interface{}) (s ClientMapping, err error) {
func parseClientMapping(v interface{}, identity string) (s *ClientMapping, err error) {
t := struct {
From string
Mapping map[string]string
}{}
if err = mapstructure.Decode(v, &t); err != nil {
return
}
s.From = t.From
s = &ClientMapping{
From: identity,
}
s.Mapping, err = parseComboMapping(t.Mapping)
return
}
@ -457,26 +485,29 @@ func (t *LocalTransport) SetHandler(handler rpc.RPCHandler) {
t.Handler = handler
}
func parsePrunes(m interface{}) (rets []Prune, err error) {
func parsePrunes(m interface{}) (rets map[string]*Prune, err error) {
asList := make([]map[string]interface{}, 0)
asList := make(map[string]map[string]interface{}, 0)
if err = mapstructure.Decode(m, &asList); err != nil {
return
}
rets = make([]Prune, len(asList))
rets = make(map[string]*Prune, len(asList))
for i, e := range asList {
if rets[i], err = parsePrune(e); err != nil {
err = fmt.Errorf("cannot parse prune job #%d: %s", i+1, err)
for name, e := range asList {
var prune *Prune
if prune, err = parsePrune(e, name); err != nil {
err = fmt.Errorf("cannot parse prune job %s: %s", name, err)
return
}
rets[name] = prune
}
return
}
func parsePrune(e map[string]interface{}) (prune Prune, err error) {
func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error) {
// Only support grid policy for now
policyName, ok := e["policy"]
if !ok || policyName != "grid" {
@ -485,7 +516,6 @@ func parsePrune(e map[string]interface{}) (prune Prune, err error) {
}
var i struct {
Name string
Grid string
DatasetFilter map[string]string `mapstructure:"dataset_filter"`
SnapshotFilter map[string]string `mapstructure:"snapshot_filter"`
@ -495,7 +525,11 @@ func parsePrune(e map[string]interface{}) (prune Prune, err error) {
return
}
prune.Name = i.Name
prune = &Prune{}
if prune.JobName, err = fullJobName(JobSectionPrune, name); err != nil {
return
}
// Parse grid policy
intervals, err := parseRetentionGridIntervalsString(i.Grid)
@ -636,30 +670,31 @@ func parseSnapshotFilter(fm map[string]string) (snapFilter zfs.FilesystemVersion
return
}
func parseAutosnaps(m interface{}) (snaps []Autosnap, err error) {
func parseAutosnaps(m interface{}) (snaps map[string]*Autosnap, err error) {
asList := make([]map[string]interface{}, 0)
if err = mapstructure.Decode(m, &asList); err != nil {
asMap := make(map[string]interface{}, 0)
if err = mapstructure.Decode(m, &asMap); err != nil {
return
}
snaps = make([]Autosnap, len(asList))
snaps = make(map[string]*Autosnap, len(asMap))
for i, e := range asList {
if snaps[i], err = parseAutosnap(e); err != nil {
err = fmt.Errorf("cannot parse autonsap job #%d: %s", i+1, err)
for name, e := range asMap {
var snap *Autosnap
if snap, err = parseAutosnap(e, name); err != nil {
err = fmt.Errorf("cannot parse autonsap job %s: %s", name, err)
return
}
snaps[name] = snap
}
return
}
func parseAutosnap(m map[string]interface{}) (a Autosnap, err error) {
func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) {
var i struct {
Name string
Prefix string
Interval string
DatasetFilter map[string]string `mapstructure:"dataset_filter"`
@ -670,7 +705,11 @@ func parseAutosnap(m map[string]interface{}) (a Autosnap, err error) {
return
}
a.Name = i.Name
a = &Autosnap{}
if a.JobName, err = fullJobName(JobSectionAutosnap, name); err != nil {
return
}
if len(i.Prefix) < 1 {
err = fmt.Errorf("prefix must not be empty")

36
cmd/prune.go
View File

@ -29,35 +29,29 @@ func init() {
func cmdPrune(cmd *cobra.Command, args []string) {
jobFailed := false
log.Printf("retending...")
for _, prune := range conf.Prunes {
if pruneArgs.job == "" || pruneArgs.job == prune.Name {
log.Printf("Beginning prune job:\n%s", prune)
ctx := PruneContext{prune, time.Now(), pruneArgs.dryRun}
err := doPrune(ctx, log)
if err != nil {
jobFailed = true
log.Printf("Prune job failed with error: %s", err)
}
log.Printf("\n")
}
if len(args) < 1 {
log.Printf("must specify exactly one job as positional argument")
os.Exit(1)
}
if jobFailed {
log.Printf("At least one job failed with an error. Check log for details.")
job, ok := conf.Prunes[args[0]]
if !ok {
log.Printf("could not find prune job: %s", args[0])
os.Exit(1)
}
log.Printf("Beginning prune job:\n%s", job)
ctx := PruneContext{job, time.Now(), pruneArgs.dryRun}
err := doPrune(ctx, log)
if err != nil {
log.Printf("Prune job failed with error: %s", err)
os.Exit(1)
}
}
type PruneContext struct {
Prune Prune
Prune *Prune
Now time.Time
DryRun bool
}

56
cmd/replication.go
View File

@ -7,6 +7,7 @@ import (
"github.com/zrepl/zrepl/rpc"
"github.com/zrepl/zrepl/zfs"
"io"
"os"
"sync"
"time"
)
@ -22,10 +23,61 @@ var RunCmd = &cobra.Command{
Run: cmdRun,
}
var PushCmd = &cobra.Command{
Use: "push",
Short: "run push job (first positional argument)",
Run: cmdPush,
}
var PullCmd = &cobra.Command{
Use: "pull",
Short: "run pull job (first positional argument)",
Run: cmdPull,
}
func init() {
RootCmd.AddCommand(RunCmd)
RunCmd.Flags().BoolVar(&runArgs.once, "once", false, "run jobs only once, regardless of configured repeat behavior")
RunCmd.Flags().StringVar(&runArgs.job, "job", "", "run only the given job")
RootCmd.AddCommand(PushCmd)
RootCmd.AddCommand(PullCmd)
}
func cmdPush(cmd *cobra.Command, args []string) {
if len(args) != 1 {
log.Printf("must specify exactly one job as positional argument")
os.Exit(1)
}
job, ok := conf.Pushs[args[0]]
if !ok {
log.Printf("could not find push job %s", args[0])
os.Exit(1)
}
if err := jobPush(job, log); err != nil {
log.Printf("error doing push: %s", err)
os.Exit(1)
}
}
func cmdPull(cmd *cobra.Command, args []string) {
if len(args) != 1 {
log.Printf("must specify exactly one job as positional argument")
os.Exit(1)
}
job, ok := conf.Pulls[args[0]]
if !ok {
log.Printf("could not find pull job %s", args[0])
os.Exit(1)
}
if err := jobPull(job, log); err != nil {
log.Printf("error doing pull: %s", err)
os.Exit(1)
}
}
func cmdRun(cmd *cobra.Command, args []string) {
@ -87,7 +139,7 @@ func cmdRun(cmd *cobra.Command, args []string) {
}
func jobPull(pull Pull, log jobrun.Logger) (err error) {
func jobPull(pull *Pull, log jobrun.Logger) (err error) {
if lt, ok := pull.From.Transport.(LocalTransport); ok {
lt.SetHandler(Handler{
@ -109,7 +161,7 @@ func jobPull(pull Pull, log jobrun.Logger) (err error) {
return doPull(PullContext{remote, log, pull.Mapping, pull.InitialReplPolicy})
}
func jobPush(push Push, log jobrun.Logger) (err error) {
func jobPush(push *Push, log jobrun.Logger) (err error) {
if _, ok := push.To.Transport.(LocalTransport); ok {
panic("no support for local pushs")

32
cmd/sampleconf/zrepl.yml
View File

@ -1,5 +1,5 @@
pools:
- name: offsite_backups
offsite_backups:
transport:
ssh:
host: 192.168.122.6
@ -8,21 +8,26 @@ pools:
identity_file: /etc/zrepl/identities/offsite_backups
pushs:
- to: offsite_backups
offsite:
to: offsite_backups
filter: {
"tank/var/db*":"ok",
"tank/usr/home*":"ok"
}
pulls:
- from: offsite_backups
offsite:
from: offsite_backups
mapping: {
# like in sinks
}
# local replication, only allowed in pull mode
# the from name 'local' is reserved for this purpose
- from: local
homemirror:
from: local
repeat:
interval: 15m
mapping: {
@ -35,21 +40,21 @@ sinks:
# 1:1 mapping of remote dataset to local dataset
# We will reject a push request which contains > 0 datasets that do not
# match a mapping
- from: db1
db1:
mapping: {
"ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data"
}
# "|" non-recursive wildcard
# the remote must present excatly one dataset, mapped to the rhs
- from: cdn_master
cdn_master:
mapping: {
"|":"tank/srv/cdn" # NOTE: | is currently an invalid character for a ZFS dataset
}
# "*" recursive wildcard
# the remote may present an arbitrary set of marks a recursive wildcard, i.e. map all remotes to a tree under rhs
- from: mirror1
mirror1:
mapping: {
"tank/foo/bar*":"zroot/backups/mirror1" # NOTE: * is currently an invalid character for a ZFS dataset
}
@ -60,7 +65,7 @@ sinks:
# local dataset (same order) or '!<space>optional reason' on stdout
# If the acceptor scripts exits with non-zero status code, the remote's
# request will be rejected
- from: complex_host
complex_host:
mapping: { #
"*":"!/path/to/acceptor" # we could just wire the path to the acceptor directly to the mapping
# but let's stick with the same type for the mapping field for now'
@ -70,7 +75,7 @@ sinks:
# Mixing the rules
# Mixing should be possible if there is a defined precedence (direct before *)
# and non-recursive wildcards are not allowed in multi-entry mapping objects
- from: special_snowflake
special_snowflake:
mapping: { # an explicit mapping mixed with a recursive wildcard
"sun/usr/home": backups/special_snowflake/homedirs,
"sun/var/db": backups/special_snowflake/database,
@ -79,8 +84,9 @@ sinks:
}
pull_acls:
# same synatx as in sinks, but the returned mapping does not matter
- from: office_backup
office_backup:
mapping: {
"tank/usr/home":"notnull"
}
@ -88,7 +94,7 @@ pull_acls:
prune:
- name: clean_backups
clean_backups:
policy: grid
grid: 6x10min | 24x1h | 7x1d | 32 x 1d | 4 x 3mon
dataset_filter: {
@ -98,7 +104,7 @@ prune:
prefix: zrepl_
}
- name: hfbak_prune # cleans up after hfbak autosnap job
hfbak_prune: # cleans up after hfbak autosnap job
policy: grid
grid: 1x1min(keep=all)
dataset_filter: {
@ -110,7 +116,7 @@ prune:
autosnap:
- name: hfbak
hfbak:
prefix: zrepl_hfbak_
interval: 1s
dataset_filter: {

36
cmd/stdinserver.go
View File

@ -11,18 +11,13 @@ import (
"os"
)
var stdinserver struct {
identity string
}
var StdinserverCmd = &cobra.Command{
Use: "stdinserver",
Use: "stdinserver CLIENT_IDENTITY",
Short: "start in stdin server mode (from authorized_keys file)",
Run: cmdStdinServer,
}
func init() {
StdinserverCmd.Flags().StringVar(&stdinserver.identity, "identity", "", "")
RootCmd.AddCommand(StdinserverCmd)
}
@ -36,37 +31,36 @@ func cmdStdinServer(cmd *cobra.Command, args []string) {
}
}()
if stdinserver.identity == "" {
err = fmt.Errorf("identity flag not set")
if len(args) != 1 || args[0] == "" {
err = fmt.Errorf("must specify client identity as positional argument")
return
}
identity := args[0]
pullACL := conf.PullACLs[identity]
if pullACL == nil {
err = fmt.Errorf("could not find PullACL for identity '%s'", identity)
return
}
identity := stdinserver.identity
var sshByteStream io.ReadWriteCloser
if sshByteStream, err = sshbytestream.Incoming(); err != nil {
return
}
findMapping := func(cm []ClientMapping, identity string) zfs.DatasetMapping {
for i := range cm {
if cm[i].From == identity {
return cm[i].Mapping
}
}
return nil
}
sinkMapping := func(identity string) (sink zfs.DatasetMapping, err error) {
if sink = findMapping(conf.Sinks, identity); sink == nil {
sinkMapping := func(identity string) (m zfs.DatasetMapping, err error) {
sink := conf.Sinks[identity]
if sink == nil {
return nil, fmt.Errorf("could not find sink for dataset")
}
return
return sink.Mapping, nil
}
sinkLogger := golog.New(logOut, fmt.Sprintf("sink[%s] ", identity), logFlags)
handler := Handler{
Logger: sinkLogger,
SinkMappingFunc: sinkMapping,
PullACL: findMapping(conf.PullACLs, identity),
PullACL: pullACL.Mapping,
}
if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil {