From 1da8f848f2f655a38824cae1f9abf522613bd0af Mon Sep 17 00:00:00 2001 From: Yannick Dylla <17772145+ydylla@users.noreply.github.com> Date: Mon, 17 Oct 2022 19:02:21 +0200 Subject: [PATCH] snapper: support custom timestamp format fixes https://github.com/zrepl/zrepl/issues/465 closes https://github.com/zrepl/zrepl/pull/639 --- config/config.go | 18 ++++---- config/config_snapshotting_test.go | 71 +++++++++++++++++++++++++++++ daemon/snapper/cron.go | 5 +- daemon/snapper/impl.go | 23 ++++++++-- daemon/snapper/periodic.go | 5 +- docs/changelog.rst | 1 + docs/configuration/snapshotting.rst | 21 +++++++++ 7 files changed, 129 insertions(+), 15 deletions(-) diff --git a/config/config.go b/config/config.go index 02a7dae..c5e7224 100644 --- a/config/config.go +++ b/config/config.go @@ -228,10 +228,11 @@ type SnapshottingEnum struct { } type SnapshottingPeriodic struct { - Type string `yaml:"type"` - Prefix string `yaml:"prefix"` - Interval time.Duration `yaml:"interval,positive"` - Hooks HookList `yaml:"hooks,optional"` + Type string `yaml:"type"` + Prefix string `yaml:"prefix"` + Interval time.Duration `yaml:"interval,positive"` + Hooks HookList `yaml:"hooks,optional"` + TimestampFormat string `yaml:"timestamp_format,optional,default=dense"` } type CronSpec struct { @@ -260,10 +261,11 @@ func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) } type SnapshottingCron struct { - Type string `yaml:"type"` - Prefix string `yaml:"prefix"` - Cron CronSpec `yaml:"cron"` - Hooks HookList `yaml:"hooks,optional"` + Type string `yaml:"type"` + Prefix string `yaml:"prefix"` + Cron CronSpec `yaml:"cron"` + Hooks HookList `yaml:"hooks,optional"` + TimestampFormat string `yaml:"timestamp_format,optional,default=dense"` } type SnapshottingManual struct { diff --git a/config/config_snapshotting_test.go b/config/config_snapshotting_test.go index 5bccf00..6bccd77 100644 --- a/config/config_snapshotting_test.go +++ b/config/config_snapshotting_test.go @@ -35,8 +35,16 @@ jobs: snapshotting: type: periodic prefix: zrepl_ + timestamp_format: dense interval: 10m ` + cron := ` + snapshotting: + type: cron + prefix: zrepl_ + timestamp_format: human + cron: "10 * * * *" +` hooks := ` snapshotting: @@ -76,6 +84,15 @@ jobs: assert.Equal(t, "periodic", snp.Type) assert.Equal(t, 10*time.Minute, snp.Interval) assert.Equal(t, "zrepl_", snp.Prefix) + assert.Equal(t, "dense", snp.TimestampFormat) + }) + + t.Run("cron", func(t *testing.T) { + c = testValidConfig(t, fillSnapshotting(cron)) + snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron) + assert.Equal(t, "cron", snp.Type) + assert.Equal(t, "zrepl_", snp.Prefix) + assert.Equal(t, "human", snp.TimestampFormat) }) t.Run("hooks", func(t *testing.T) { @@ -88,3 +105,57 @@ jobs: }) } + +func TestSnapshottingTimestampDefaults(t *testing.T) { + tmpl := ` +jobs: +- name: foo + type: push + connect: + type: local + listener_name: foo + client_identity: bar + filesystems: {"<": true} + %s + pruning: + keep_sender: + - type: last_n + count: 10 + keep_receiver: + - type: last_n + count: 10 +` + + periodic := ` + snapshotting: + type: periodic + prefix: zrepl_ + interval: 10m +` + cron := ` + snapshotting: + type: cron + prefix: zrepl_ + cron: "10 * * * *" +` + + fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) } + var c *Config + + t.Run("periodic", func(t *testing.T) { + c = testValidConfig(t, fillSnapshotting(periodic)) + snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic) + assert.Equal(t, "periodic", snp.Type) + assert.Equal(t, 10*time.Minute, snp.Interval) + assert.Equal(t, "zrepl_", snp.Prefix) + assert.Equal(t, "dense", snp.TimestampFormat) // default was set correctly + }) + + t.Run("cron", func(t *testing.T) { + c = testValidConfig(t, fillSnapshotting(cron)) + snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron) + assert.Equal(t, "cron", snp.Type) + assert.Equal(t, "zrepl_", snp.Prefix) + assert.Equal(t, "dense", snp.TimestampFormat) // default was set correctly + }) +} diff --git a/daemon/snapper/cron.go b/daemon/snapper/cron.go index 520f7a6..8d50e41 100644 --- a/daemon/snapper/cron.go +++ b/daemon/snapper/cron.go @@ -20,8 +20,9 @@ func cronFromConfig(fsf zfs.DatasetFilter, in config.SnapshottingCron) (*Cron, e return nil, errors.Wrap(err, "hook config error") } planArgs := planArgs{ - prefix: in.Prefix, - hooks: hooksList, + prefix: in.Prefix, + timestampFormat: in.TimestampFormat, + hooks: hooksList, } return &Cron{config: in, fsf: fsf, planArgs: planArgs}, nil } diff --git a/daemon/snapper/impl.go b/daemon/snapper/impl.go index 09f21d4..6d2cec6 100644 --- a/daemon/snapper/impl.go +++ b/daemon/snapper/impl.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sort" + "strconv" "strings" "time" @@ -14,8 +15,9 @@ import ( ) type planArgs struct { - prefix string - hooks *hooks.List + prefix string + timestampFormat string + hooks *hooks.List } type plan struct { @@ -58,6 +60,21 @@ type snapProgress struct { runResults hooks.PlanReport } +func (plan *plan) formatNow(format string) string { + now := time.Now().UTC() + switch strings.ToLower(format) { + case "dense": + format = "20060102_150405_000" + case "human": + format = "2006-01-02_15:04:05" + case "iso-8601": + format = "2006-01-02T15:04:05.000Z" + case "unix-seconds": + return strconv.FormatInt(now.Unix(), 10) + } + return now.Format(format) +} + func (plan *plan) execute(ctx context.Context, dryRun bool) (ok bool) { hookMatchCount := make(map[hooks.Hook]int, len(*plan.args.hooks)) @@ -68,7 +85,7 @@ func (plan *plan) execute(ctx context.Context, dryRun bool) (ok bool) { anyFsHadErr := false // TODO channel programs -> allow a little jitter? for fs, progress := range plan.snaps { - suffix := time.Now().In(time.UTC).Format("20060102_150405_000") + suffix := plan.formatNow(plan.args.timestampFormat) snapname := fmt.Sprintf("%s%s", plan.args.prefix, suffix) ctx := logging.WithInjectedField(ctx, "fs", fs.ToString()) diff --git a/daemon/snapper/periodic.go b/daemon/snapper/periodic.go index 85e6824..1a5c27e 100644 --- a/daemon/snapper/periodic.go +++ b/daemon/snapper/periodic.go @@ -35,8 +35,9 @@ func periodicFromConfig(g *config.Global, fsf zfs.DatasetFilter, in *config.Snap interval: in.Interval, fsf: fsf, planArgs: planArgs{ - prefix: in.Prefix, - hooks: hookList, + prefix: in.Prefix, + timestampFormat: in.TimestampFormat, + hooks: hookList, }, // ctx and log is set in Run() } diff --git a/docs/changelog.rst b/docs/changelog.rst index 77b8a36..4d369b7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -22,6 +22,7 @@ Developers should consult the git commit log or GitHub issue tracker. * `Feature Wishlist on GitHub `_ * |feature| :ref:`Schedule-based snapshotting` using ``cron`` syntax instead of an interval. +* |feature| Configurable timestamp format for snapshot names via :ref:`timestamp_format`. * |feature| Add ``ZREPL_DESTROY_MAX_BATCH_SIZE`` env var (default 0=unlimited). * |bugfix| Fix resuming from interrupted replications that use ``send.raw`` on unencrypted datasets. diff --git a/docs/configuration/snapshotting.rst b/docs/configuration/snapshotting.rst index a09e13d..30885ef 100644 --- a/docs/configuration/snapshotting.rst +++ b/docs/configuration/snapshotting.rst @@ -62,6 +62,9 @@ The ``periodic`` and ``cron`` snapshotting types share some common options and b type: periodic prefix: zrepl_ interval: 10m + # Timestamp format that is used as snapshot suffix. + # Can be any of "dense" (default), "human", "iso-8601", "unix-seconds" or a custom Go time format (see https://go.dev/src/time/format.go) + timestamp_format: dense hooks: ... pruning: ... @@ -91,6 +94,9 @@ The snapshotter uses the ``prefix`` to identify which snapshots it created. # (second, optional) minute hour day-of-month month day-of-week # This example takes snapshots daily at 3:00. cron: "0 3 * * *" + # Timestamp format that is used as snapshot suffix. + # Can be any of "dense" (default), "human", "iso-8601", "unix-seconds" or a custom Go time format (see https://go.dev/src/time/format.go) + timestamp_format: dense pruning: ... In ``cron`` mode, the snapshotter takes snaphots at fixed points in time. @@ -98,6 +104,21 @@ See https://en.wikipedia.org/wiki/Cron for details on the syntax. zrepl uses the ``the github.com/robfig/cron/v3`` Go package for parsing. An optional field for "seconds" is supported to take snapshots at sub-minute frequencies. +.. _job-snapshotting-timestamp_format: + +Timestamp Format +~~~~~~~~~~~~~~~~ + +The ``cron`` and ``periodic`` snapshotter support configuring a custom timestamp format that is used as suffix for the snapshot name. +It can be used by setting ``timestamp_format`` to any of the following values: + +* ``dense`` (default) looks like ``20060102_150405_000`` +* ``human`` looks like ``2006-01-02_15:04:05`` +* ``iso-8601`` looks like ``2006-01-02T15:04:05.000Z`` +* ``unix-seconds`` looks like ``1136214245`` +* Any custom Go time format accepted by `time.Time#Format `_. + + ``manual`` Snapshotting -----------------------