From b1f8cdf38524bbce6d330935ae2a99071997516b Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 31 Aug 2020 14:05:32 +0200 Subject: [PATCH] [#373] pruning: add optional `regex` field to `last_n` rule fixes #373 --- config/config.go | 1 + docs/configuration/prune.rst | 4 ++- pruning/keep_last_n.go | 40 +++++++++++++++++++++++------- pruning/keep_last_n_test.go | 47 ++++++++++++++++++++++++++++++------ pruning/pruning.go | 2 +- 5 files changed, 76 insertions(+), 18 deletions(-) diff --git a/config/config.go b/config/config.go index 117f711..9bbf1b3 100644 --- a/config/config.go +++ b/config/config.go @@ -324,6 +324,7 @@ type PruneKeepNotReplicated struct { type PruneKeepLastN struct { Type string `yaml:"type"` Count int `yaml:"count"` + Regex string `yaml:"regex,optional"` } type PruneKeepRegex struct { // FIXME rename to KeepRegex diff --git a/docs/configuration/prune.rst b/docs/configuration/prune.rst index dac49fb..4abc216 100644 --- a/docs/configuration/prune.rst +++ b/docs/configuration/prune.rst @@ -175,9 +175,11 @@ Policy ``last_n`` keep_receiver: - type: last_n count: 10 + regex: ^zrepl_.*$ # optional ... -``last_n`` keeps the last ``count`` snapshots (last = youngest = most recent creation date). +``last_n`` filters the snapshot list by ``regex``, then keeps the last ``count`` snapshots in that list (last = youngest = most recent creation date) +All snapshots that don't match ``regex`` or exceed ``count`` in the filtered list are destroyed unless matched by other rules. .. _prune-keep-regex: diff --git a/pruning/keep_last_n.go b/pruning/keep_last_n.go index 3f90d8a..6bd7e2d 100644 --- a/pruning/keep_last_n.go +++ b/pruning/keep_last_n.go @@ -1,6 +1,7 @@ package pruning import ( + "regexp" "sort" "strings" @@ -8,14 +9,27 @@ import ( ) type KeepLastN struct { - n int + n int + re *regexp.Regexp } -func NewKeepLastN(n int) (*KeepLastN, error) { +func MustKeepLastN(n int, regex string) *KeepLastN { + k, err := NewKeepLastN(n, regex) + if err != nil { + panic(err) + } + return k +} + +func NewKeepLastN(n int, regex string) (*KeepLastN, error) { if n <= 0 { return nil, errors.Errorf("must specify positive number as 'keep last count', got %d", n) } - return &KeepLastN{n}, nil + re, err := regexp.Compile(regex) + if err != nil { + return nil, errors.Errorf("invalid regex %q: %s", regex, err) + } + return &KeepLastN{n, re}, nil } func (k KeepLastN) KeepRule(snaps []Snapshot) (destroyList []Snapshot) { @@ -24,17 +38,25 @@ func (k KeepLastN) KeepRule(snaps []Snapshot) (destroyList []Snapshot) { return []Snapshot{} } - res := shallowCopySnapList(snaps) + matching, notMatching := partitionSnapList(snaps, func(snapshot Snapshot) bool { + return k.re.MatchString(snapshot.Name()) + }) + // snaps that don't match the regex are not kept by this rule + destroyList = append(destroyList, notMatching...) - sort.Slice(res, func(i, j int) bool { + if len(matching) == 0 { + return destroyList + } + + sort.Slice(matching, func(i, j int) bool { // by date (youngest first) - id, jd := res[i].Date(), res[j].Date() + id, jd := matching[i].Date(), matching[j].Date() if !id.Equal(jd) { return id.After(jd) } // then lexicographically descending (e.g. b, a) - return strings.Compare(res[i].Name(), res[j].Name()) == 1 + return strings.Compare(matching[i].Name(), matching[j].Name()) == 1 }) - - return res[k.n:] + destroyList = append(destroyList, matching[k.n:]...) + return destroyList } diff --git a/pruning/keep_last_n_test.go b/pruning/keep_last_n_test.go index cc3fcf4..c96bbb9 100644 --- a/pruning/keep_last_n_test.go +++ b/pruning/keep_last_n_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestKeepLastN(t *testing.T) { @@ -28,7 +29,7 @@ func TestKeepLastN(t *testing.T) { "keep2": { inputs: inputs["s1"], rules: []KeepRule{ - KeepLastN{2}, + MustKeepLastN(2, ""), }, expDestroy: map[string]bool{ "1": true, "2": true, "3": true, @@ -37,34 +38,66 @@ func TestKeepLastN(t *testing.T) { "keep1OfTwoWithSameTime": { // Keep one of two with same time inputs: inputs["s1"], rules: []KeepRule{ - KeepLastN{1}, + MustKeepLastN(1, ""), }, expDestroy: map[string]bool{"1": true, "2": true, "3": true, "4": true}, }, "keepMany": { inputs: inputs["s1"], rules: []KeepRule{ - KeepLastN{100}, + MustKeepLastN(100, ""), }, expDestroy: map[string]bool{}, }, - "empty": { + "empty_input": { inputs: inputs["s2"], rules: []KeepRule{ - KeepLastN{100}, + MustKeepLastN(100, ""), }, expDestroy: map[string]bool{}, }, + "empty_regex": { + inputs: inputs["s1"], + rules: []KeepRule{ + MustKeepLastN(4, ""), + }, + expDestroy: map[string]bool{ + "1": true, + }, + }, + "multiple_regexes": { + inputs: []Snapshot{ + stubSnap{"a1", false, o(10)}, + stubSnap{"b1", false, o(11)}, + stubSnap{"a2", false, o(20)}, + stubSnap{"b2", false, o(21)}, + stubSnap{"a3", false, o(30)}, + stubSnap{"b3", false, o(31)}, + }, + rules: []KeepRule{ + MustKeepLastN(2, "^a"), + MustKeepLastN(2, "^b"), + }, + expDestroy: map[string]bool{ + "a1": true, + "b1": true, + }, + }, } testTable(tcs, t) t.Run("mustBePositive", func(t *testing.T) { var err error - _, err = NewKeepLastN(0) + _, err = NewKeepLastN(0, "foo") assert.Error(t, err) - _, err = NewKeepLastN(-5) + _, err = NewKeepLastN(-5, "foo") assert.Error(t, err) }) + t.Run("emptyRegexAllowed", func(t *testing.T) { + _, err := NewKeepLastN(23, "") + require.NoError(t, err) + }) + } diff --git a/pruning/pruning.go b/pruning/pruning.go index 9f36581..60a4e6d 100644 --- a/pruning/pruning.go +++ b/pruning/pruning.go @@ -60,7 +60,7 @@ func RuleFromConfig(in config.PruningEnum) (KeepRule, error) { case *config.PruneKeepNotReplicated: return NewKeepNotReplicated(), nil case *config.PruneKeepLastN: - return NewKeepLastN(v.Count) + return NewKeepLastN(v.Count, v.Regex) case *config.PruneKeepRegex: return NewKeepRegex(v.Regex, v.Negate) case *config.PruneGrid: