From 428a60870ab8018aa25781b69d0755c64f3cb154 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 30 Aug 2020 22:42:40 +0200 Subject: [PATCH] pruning: cleanup retention grid impl + tests + correct docs package is now at 95% code coverage and the additional tests codify all behavior specified in the docs There is a slight change in behavior: Intervals are now [duration) instead of (duration]. If the leftmost interval is not keep=all, the most recently created snapshot will be destroyed if there are other snapshots within that first interval. Since we recommend keep=all all over the docs, and zrepl 0.3 will put holds on that snapshot if it is being replicated, I feel like this is an acceptable change in behavior. refs #292 fixup of 0bbe2befcec8eef3c61acec8be762f1e1293a96d --- docs/configuration/prune.rst | 43 +++--- pruning/keep_grid.go | 31 +--- pruning/retentiongrid/retentiongrid.go | 150 +++++++++++++------- pruning/retentiongrid/retentiongrid_test.go | 138 ++++++++++-------- 4 files changed, 206 insertions(+), 156 deletions(-) diff --git a/docs/configuration/prune.rst b/docs/configuration/prune.rst index fecc658..dac49fb 100644 --- a/docs/configuration/prune.rst +++ b/docs/configuration/prune.rst @@ -93,22 +93,23 @@ Policy ``grid`` ... The retention grid can be thought of as a time-based sieve that thins out snapshots as they get older. + The ``grid`` field specifies a list of adjacent time intervals. +Each interval is a bucket with a maximum capacity of ``keep`` snapshots. The following procedure happens during pruning: #. The list of snapshots is filtered by the regular expression in ``regex``. - Only snapshots names that match the regex are considered for this rule, all others are not affected. -#. The filtered list of snapshots is sorted by ``creation``. -#. The left edge of the first interval is aligned to the ``creation`` date of the youngest snapshot. -#. A list of buckets is created, one for each interval. -#. The snapshots are placed into the bucket that matches their ``creation`` date. -#. For each bucket + Only snapshots names that match the regex are considered for this rule, all others will be pruned unless another rule keeps them. +#. The snapshots that match ``regex`` are placed onto a time axis according to their ``creation`` date. + The youngest snapshot is on the left, the oldest on the right. +#. The first buckets are placed "under" that axis so that the ``grid`` spec's first bucket's left edge aligns with youngest snapshot. +#. All subsequent buckets are placed adjacent to their predecessor bucket. +#. Now each snapshot on the axis either falls into one bucket or it is older than our rightmost bucket. + Buckets are left-inclusive and right-exclusive which means that a snapshot on the edge of bucket will always 'fall into the right one'. +#. Snapshots older than the rightmost bucket **not kept** by this gridspec. +#. For each bucket, we only keep the ``keep`` oldest snapshots. - #. the contained snapshot list is sorted by creation. - #. snapshots from the list, oldest first, are destroyed until the specified ``keep`` count is reached. - #. all remaining snapshots on the list are kept. - -The syntax to describe the list of time intervals ("buckets") is as follows: +The syntax to describe the bucket list is as follows: :: @@ -131,30 +132,30 @@ The syntax to describe the list of time intervals ("buckets") is as follows: regex: .* ` - 0h 1h 2h 3h 4h 5h 6h 7h 8h - | | | | | | | | | - |-Bucket 1-|------Bucket 2-------|-------Bucket 3------|------------Bucket 4------------| - | keep=all | keep=1 | keep=1 | keep=1 | + 0h 1h 2h 3h 4h 5h 6h 7h 8h 9h + | | | | | | | | | | + |-Bucket1-|-----Bucket 2------|------Bucket 3-----|-----------Bucket 4----------| + | keep=all| keep=1 | keep=1 | keep=1 | - Let us consider the following set of snapshots @a-zA-C , taken at an interval of ~15min: + Let us consider the following set of snapshots @a-zA-C: - | a b c d e f g h i j k l m n o p q r s t u v w x y z A B C | - + | a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D | The `grid` algorithm maps them to their respective buckets: Bucket 1: a, b, c Bucket 2: d,e,f,g,h,i,j - Bucket 3: k,l,m,n,o,p,q,r - Bucket 4: q,r,s,t,u,v,w,x,y,z,A,B,C + Bucket 3: k,l,m,n,o,p + Bucket 4: q,r, q,r,s,t,u,v,w,x,y,z + None: A,B,C,D It then applies the per-bucket pruning logic described above which resulting in the following list of remaining snapshots. - | a b c d k s | + | a b c j p z | Note that it only makes sense to grow (not shorten) the interval duration for buckets further in the past since each bucket acts like a low-pass filter for incoming snapshots diff --git a/pruning/keep_grid.go b/pruning/keep_grid.go index 2dffc07..92f6515 100644 --- a/pruning/keep_grid.go +++ b/pruning/keep_grid.go @@ -3,7 +3,6 @@ package pruning import ( "fmt" "regexp" - "sort" "time" "github.com/pkg/errors" @@ -88,14 +87,6 @@ func newKeepGrid(re *regexp.Regexp, configIntervals []config.RetentionInterval) }, nil } -type retentionGridAdaptor struct { - Snapshot -} - -func (a retentionGridAdaptor) LessThan(b retentiongrid.Entry) bool { - return a.Date().Before(b.Date()) -} - // Prune filters snapshots with the retention grid. func (p *KeepGrid) KeepRule(snaps []Snapshot) (destroyList []Snapshot) { @@ -110,24 +101,16 @@ func (p *KeepGrid) KeepRule(snaps []Snapshot) (destroyList []Snapshot) { return destroyList } - // Build adaptors for retention grid - adaptors := make([]retentiongrid.Entry, 0) - for i := range matching { - adaptors = append(adaptors, retentionGridAdaptor{matching[i]}) - } - - // determine 'now' edge - sort.SliceStable(adaptors, func(i, j int) bool { - return adaptors[i].LessThan(adaptors[j]) - }) - now := adaptors[len(adaptors)-1].Date() - // Evaluate retention grid - _, removea := p.retentionGrid.FitEntries(now, adaptors) + entrySlice := make([]retentiongrid.Entry, 0) + for i := range matching { + entrySlice = append(entrySlice, matching[i]) + } + _, gridDestroyList := p.retentionGrid.FitEntries(entrySlice) // Revert adaptors - for i := range removea { - destroyList = append(destroyList, removea[i].(retentionGridAdaptor).Snapshot) + for i := range gridDestroyList { + destroyList = append(destroyList, gridDestroyList[i].(Snapshot)) } return destroyList } diff --git a/pruning/retentiongrid/retentiongrid.go b/pruning/retentiongrid/retentiongrid.go index 78f8767..baf23d2 100644 --- a/pruning/retentiongrid/retentiongrid.go +++ b/pruning/retentiongrid/retentiongrid.go @@ -16,85 +16,127 @@ type Grid struct { intervals []Interval } -//A point inside the grid, i.e. a thing the grid can decide to remove type Entry interface { Date() time.Time - LessThan(b Entry) bool -} - -func dateInInterval(date, startDateInterval time.Time, i Interval) bool { - return date.After(startDateInterval) && date.Before(startDateInterval.Add(i.Length())) } func NewGrid(l []Interval) *Grid { + if len(l) == 0 { + panic("must specify at least one interval") + } // TODO Maybe check for ascending interval lengths here, although the algorithm // itself doesn't care about that. return &Grid{l} } -// Partition a list of RetentionGridEntries into the Grid, -// relative to a given start date `now`. -// -// The `keepCount` oldest entries per `retentiongrid.Interval` are kept (`keep`), -// the others are removed (`remove`). -// -// Entries that are younger than `now` are always kept. -// Those that are older than the earliest beginning of an interval are removed. -func (g Grid) FitEntries(now time.Time, entries []Entry) (keep, remove []Entry) { +func (g Grid) FitEntries(entries []Entry) (keep, remove []Entry) { - type bucket struct { - entries []Entry + if len(entries) == 0 { + return } + + // determine 'now' based on youngest snapshot + // => sort youngest-to-oldest + sort.SliceStable(entries, func(i, j int) bool { + return entries[i].Date().After(entries[j].Date()) + }) + now := entries[0].Date() + + return g.fitEntriesWithNow(now, entries) +} + +type bucket struct { + keepCount int + youngerThan time.Time + olderThanOrEq time.Time + entries []Entry +} + +func makeBucketFromInterval(olderThanOrEq time.Time, i Interval) bucket { + var b bucket + kc := i.KeepCount() + if kc == 0 { + panic("keep count 0 is not allowed") + } + if (kc < 0) && kc != RetentionGridKeepCountAll { + panic("negative keep counts are not allowed") + } + b.keepCount = kc + b.olderThanOrEq = olderThanOrEq + b.youngerThan = b.olderThanOrEq.Add(-i.Length()) + return b +} + +func (b *bucket) Contains(e Entry) bool { + d := e.Date() + olderThan := d.Before(b.olderThanOrEq) + eq := d.Equal(b.olderThanOrEq) + youngerThan := d.After(b.youngerThan) + return (olderThan || eq) && youngerThan +} + +func (b *bucket) AddIfContains(e Entry) (added bool) { + added = b.Contains(e) + if added { + b.entries = append(b.entries, e) + } + return +} + +func (b *bucket) RemoveYoungerSnapsExceedingKeepCount() (removed []Entry) { + + if b.keepCount == RetentionGridKeepCountAll { + return nil + } + + removeCount := len(b.entries) - b.keepCount + if removeCount <= 0 { + return nil + } + + // sort youngest-to-oldest + sort.SliceStable(b.entries, func(i, j int) bool { + return b.entries[i].Date().After(b.entries[j].Date()) + }) + + return b.entries[:removeCount] +} + +func (g Grid) fitEntriesWithNow(now time.Time, entries []Entry) (keep, remove []Entry) { + buckets := make([]bucket, len(g.intervals)) + buckets[0] = makeBucketFromInterval(now, g.intervals[0]) + for i := 1; i < len(g.intervals); i++ { + buckets[i] = makeBucketFromInterval(buckets[i-1].youngerThan, g.intervals[i]) + } + keep = make([]Entry, 0) remove = make([]Entry, 0) - oldestIntervalStart := now - for i := range g.intervals { - oldestIntervalStart = oldestIntervalStart.Add(-g.intervals[i].Length()) - } - +assignEntriesToBuckets: for ei := 0; ei < len(entries); ei++ { e := entries[ei] - - date := e.Date() - - if date == now || date.After(now) { + // unconditionally keep entries that are in the future + if now.Before(e.Date()) { keep = append(keep, e) - continue - } else if date.Before(oldestIntervalStart) { - remove = append(remove, e) - continue + continue assignEntriesToBuckets } - - iStartTime := now - for i := 0; i < len(g.intervals); i++ { - iStartTime = iStartTime.Add(-g.intervals[i].Length()) - if date == iStartTime || dateInInterval(date, iStartTime, g.intervals[i]) { - buckets[i].entries = append(buckets[i].entries, e) + // add to matching bucket, if any + for bi := range buckets { + if buckets[bi].AddIfContains(e) { + continue assignEntriesToBuckets } } + // unconditionally remove entries older than the oldest bucket + remove = append(remove, e) } - for bi, b := range buckets { - - interval := g.intervals[bi] - - sort.SliceStable(b.entries, func(i, j int) bool { - return b.entries[i].LessThan((b.entries[j])) - }) - - i := 0 - for ; (interval.KeepCount() == RetentionGridKeepCountAll || i < interval.KeepCount()) && i < len(b.entries); i++ { - keep = append(keep, b.entries[i]) - } - for ; i < len(b.entries); i++ { - remove = append(remove, b.entries[i]) - } - + // now apply the `KeepCount` per bucket + for _, b := range buckets { + destroy := b.RemoveYoungerSnapsExceedingKeepCount() + remove = append(remove, destroy...) + keep = append(keep, b.entries...) } - return - } diff --git a/pruning/retentiongrid/retentiongrid_test.go b/pruning/retentiongrid/retentiongrid_test.go index c487f13..e885d69 100644 --- a/pruning/retentiongrid/retentiongrid_test.go +++ b/pruning/retentiongrid/retentiongrid_test.go @@ -10,25 +10,18 @@ import ( "github.com/stretchr/testify/assert" ) -type retentionIntervalStub struct { +type testInterval struct { length time.Duration keepCount int } -func (i *retentionIntervalStub) Length() time.Duration { - return i.length -} - -func (i *retentionIntervalStub) KeepCount() int { - return i.keepCount -} +func (i *testInterval) Length() time.Duration { return i.length } +func (i *testInterval) KeepCount() int { return i.keepCount } func gridFromString(gs string) (g *Grid) { - intervals := strings.Split(gs, "|") - g = &Grid{ - intervals: make([]Interval, len(intervals)), - } - for idx, i := range intervals { + sintervals := strings.Split(gs, "|") + intervals := make([]Interval, len(sintervals)) + for idx, i := range sintervals { comps := strings.SplitN(i, ",", 2) var durationStr, numSnapsStr string durationStr = comps[0] @@ -39,7 +32,7 @@ func gridFromString(gs string) (g *Grid) { } var err error - var interval retentionIntervalStub + var interval testInterval if interval.keepCount, err = strconv.Atoi(numSnapsStr); err != nil { panic(err) @@ -48,44 +41,38 @@ func gridFromString(gs string) (g *Grid) { panic(err) } - g.intervals[idx] = &interval + intervals[idx] = &interval } - return + return NewGrid(intervals) } -type dummySnap struct { +type testSnap struct { Name string ShouldKeep bool date time.Time } -func (ds dummySnap) Date() time.Time { - return ds.date -} - -func (ds dummySnap) LessThan(b Entry) bool { - return ds.date.Before(b.(dummySnap).date) // don't have a txg here -} +func (ds testSnap) Date() time.Time { return ds.date } func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, remove []Entry) { - snapDescr := func(d dummySnap) string { + snapDescr := func(d testSnap) string { return fmt.Sprintf("%s@%s", d.Name, d.date.Sub(now)) } t.Logf("keep list:\n") for k := range keep { - t.Logf("\t%s\n", snapDescr(keep[k].(dummySnap))) + t.Logf("\t%s\n", snapDescr(keep[k].(testSnap))) } t.Logf("remove list:\n") for k := range remove { - t.Logf("\t%s\n", snapDescr(remove[k].(dummySnap))) + t.Logf("\t%s\n", snapDescr(remove[k].(testSnap))) } t.Logf("\n\n") for _, s := range input { - d := s.(dummySnap) + d := s.(testSnap) descr := snapDescr(d) t.Logf("testing %s\n", descr) if d.ShouldKeep { @@ -97,21 +84,18 @@ func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, r t.Logf("resulting list:\n") for k := range keep { - t.Logf("\t%s\n", snapDescr(keep[k].(dummySnap))) + t.Logf("\t%s\n", snapDescr(keep[k].(testSnap))) } } -func TestRetentionGridFitEntriesEmptyInput(t *testing.T) { +func TestEmptyInput(t *testing.T) { g := gridFromString("10m|10m|10m|1h") - keep, remove := g.FitEntries(time.Now(), []Entry{}) + keep, remove := g.FitEntries([]Entry{}) assert.Empty(t, keep) assert.Empty(t, remove) } -func TestRetentionGridFitEntriesIntervalBoundariesAndAlignment(t *testing.T) { - - // Intervals are (duration], i.e. 10min is in the first interval, not in the second - +func TestIntervalBoundariesAndAlignment(t *testing.T) { g := gridFromString("10m|10m|10m") t.Logf("%#v\n", g) @@ -119,20 +103,51 @@ func TestRetentionGridFitEntriesIntervalBoundariesAndAlignment(t *testing.T) { now := time.Unix(0, 0) snaps := []Entry{ - dummySnap{"0", true, now.Add(1 * time.Minute)}, // before now - dummySnap{"1", true, now}, // before now - dummySnap{"2", true, now.Add(-10 * time.Minute)}, // 1st interval - dummySnap{"3", true, now.Add(-20 * time.Minute)}, // 2nd interval - dummySnap{"4", true, now.Add(-30 * time.Minute)}, // 3rd interval - dummySnap{"5", false, now.Add(-40 * time.Minute)}, // after last interval + testSnap{"0", true, now.Add(1 * time.Minute)}, // before now => keep unconditionally + testSnap{"1", true, now}, // 1st interval left edge => inclusive + testSnap{"2", true, now.Add(-10 * time.Minute)}, // 2nd interval left edge => inclusive + testSnap{"3", true, now.Add(-20 * time.Minute)}, // 3rd interval left edge => inclusuive + testSnap{"4", false, now.Add(-30 * time.Minute)}, // 3rd interval right edge => excludive + testSnap{"5", false, now.Add(-40 * time.Minute)}, // after last interval => remove unconditionally } - keep, remove := g.FitEntries(now, snaps) + keep, remove := g.fitEntriesWithNow(now, snaps) validateRetentionGridFitEntries(t, now, snaps, keep, remove) - } -func TestRetentionGridFitEntries(t *testing.T) { +func TestKeepsOldestSnapsInABucket(t *testing.T) { + g := gridFromString("1m,2") + + relt := func(secs int64) time.Time { return time.Unix(secs, 0) } + + snaps := []Entry{ + testSnap{"1", true, relt(1)}, + testSnap{"2", true, relt(2)}, + testSnap{"3", false, relt(3)}, + testSnap{"4", false, relt(4)}, + testSnap{"5", false, relt(5)}, + } + + now := relt(6) + keep, remove := g.FitEntries(snaps) + validateRetentionGridFitEntries(t, now, snaps, keep, remove) +} + +func TestRespectsKeepCountAll(t *testing.T) { + g := gridFromString("1m,-1|1m,1") + relt := func(secs int64) time.Time { return time.Unix(secs, 0) } + snaps := []Entry{ + testSnap{"a", true, relt(0)}, + testSnap{"b", true, relt(-1)}, + testSnap{"c", true, relt(-2)}, + testSnap{"d", false, relt(-60)}, + testSnap{"e", true, relt(-61)}, + } + keep, remove := g.FitEntries(snaps) + validateRetentionGridFitEntries(t, relt(61), snaps, keep, remove) +} + +func TestComplex(t *testing.T) { g := gridFromString("10m,-1|10m|10m,2|1h") @@ -141,20 +156,29 @@ func TestRetentionGridFitEntries(t *testing.T) { now := time.Unix(0, 0) snaps := []Entry{ - dummySnap{"1", true, now.Add(3 * time.Minute)}, // pre-now must always be kept - dummySnap{"b1", true, now.Add(-6 * time.Minute)}, // 1st interval allows unlimited entries - dummySnap{"b3", true, now.Add(-8 * time.Minute)}, // 1st interval allows unlimited entries - dummySnap{"b2", true, now.Add(-9 * time.Minute)}, // 1st interval allows unlimited entries - dummySnap{"a", false, now.Add(-11 * time.Minute)}, - dummySnap{"c", true, now.Add(-19 * time.Minute)}, // 2nd interval allows 1 entry - dummySnap{"foo", false, now.Add(-25 * time.Minute)}, - dummySnap{"bar", true, now.Add(-26 * time.Minute)}, // 3rd interval allows 2 entries - dummySnap{"border", true, now.Add(-30 * time.Minute)}, - dummySnap{"d", true, now.Add(-1*time.Hour - 15*time.Minute)}, - dummySnap{"e", false, now.Add(-1*time.Hour - 31*time.Minute)}, // before earliest interval must always be deleted - dummySnap{"f", false, now.Add(-2 * time.Hour)}, + // pre-now must always be kept + testSnap{"1", true, now.Add(3 * time.Minute)}, + // 1st interval allows unlimited entries + testSnap{"b1", true, now.Add(-6 * time.Minute)}, + testSnap{"b3", true, now.Add(-8 * time.Minute)}, + testSnap{"b2", true, now.Add(-9 * time.Minute)}, + // 2nd interval allows 1 entry + testSnap{"a", false, now.Add(-11 * time.Minute)}, + testSnap{"c", true, now.Add(-19 * time.Minute)}, + // 3rd interval allows 2 entries + testSnap{"foo", true, now.Add(-25 * time.Minute)}, + testSnap{"bar", true, now.Add(-26 * time.Minute)}, + // this is at the left edge of the 4th interval + testSnap{"border", false, now.Add(-30 * time.Minute)}, + // right in the 4th interval + testSnap{"d", true, now.Add(-1*time.Hour - 15*time.Minute)}, + // on the right edge of 4th interval => not in it => delete + testSnap{"q", false, now.Add(-1*time.Hour - 30*time.Minute)}, + // older then 4th interval => always delete + testSnap{"e", false, now.Add(-1*time.Hour - 31*time.Minute)}, + testSnap{"f", false, now.Add(-2 * time.Hour)}, } - keep, remove := g.FitEntries(now, snaps) + keep, remove := g.fitEntriesWithNow(now, snaps) validateRetentionGridFitEntries(t, now, snaps, keep, remove)