fs: Add --track-renames-strategy for configurable matching criteria for --track-renames

This commit adds the `--track-renames-strategy` flag which allows the
user to choose the strategy for tracking renames when using the
`--track-renames` flag.

This can be "hash" or "modtime" or both currently.

This, when used with `--track-renames-strategy modtime` enables
support for tracking renames in encrypted remotes.

Fixes #3696
Fixes #2721

docs/content/docs.md

@@ -1153,6 +1153,14 @@ Note also that `--track-renames` is incompatible with
 `--delete-before` and will select `--delete-after` instead of
 `--delete-during`.
 
+### --track-renames-strategy (hash,modtime) ###
+
+This option changes the matching criteria for `--track-renames` to match
+by any combination of modtime, hash, size. Matchig by size is always enabled
+no matter what option is selected here. This also means
+that it enables `--track-renames` support for encrypted destinations.
+If nothing is specified, the default option is matching by hashes.
+
 ### --delete-(before,during,after) ###
 
 This option allows you to specify when files on your destination are

docs/content/flags.md

@@ -128,6 +128,7 @@ These flags are available for every command.
       --tpslimit float                       Limit HTTP transactions per second to this.
       --tpslimit-burst int                   Max burst of transactions for --tpslimit. (default 1)
       --track-renames                        When synchronizing, track file renames and do a server side move if possible
+      --track-renames-strategy                  When tracking renames, use multiple strategies of hash, modtime
       --transfers int                        Number of file transfers to run in parallel. (default 4)
   -u, --update                               Skip files that are newer on the destination.
       --use-cookies                          Enable session cookiejar.

fs/config.go

@@ -60,6 +60,7 @@ type ConfigInfo struct {
 	DeleteMode             DeleteMode
 	MaxDelete              int64
 	TrackRenames           bool   // Track file renames.
+	TrackRenamesStrategy   string // Comma separated list of stratgies used to track renames
 	LowLevelRetries        int
 	UpdateOlder            bool // Skip files that are newer on the destination
 	NoGzip                 bool // Disable compression
@@ -145,6 +146,8 @@ func NewConfig() *ConfigInfo {
 	c.MultiThreadCutoff = SizeSuffix(250 * 1024 * 1024)
 	c.MultiThreadStreams = 4
 
+	c.TrackRenamesStrategy = "hash"
+
 	return c
 }
 

fs/config/configflags/configflags.go

@@ -61,6 +61,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transferring (default)")
 	flags.Int64VarP(flagSet, &fs.Config.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes")
 	flags.BoolVarP(flagSet, &fs.Config.TrackRenames, "track-renames", "", fs.Config.TrackRenames, "When synchronizing, track file renames and do a server side move if possible")
+	flags.StringVarP(flagSet, &fs.Config.TrackRenamesStrategy, "track-renames-strategy", "", fs.Config.TrackRenamesStrategy, "Strategies to use when synchronizing using track-renames hash|modtime")
 	flags.IntVarP(flagSet, &fs.Config.LowLevelRetries, "low-level-retries", "", fs.Config.LowLevelRetries, "Number of low level retries to do.")
 	flags.BoolVarP(flagSet, &fs.Config.UpdateOlder, "update", "u", fs.Config.UpdateOlder, "Skip files that are newer on the destination.")
 	flags.BoolVarP(flagSet, &fs.Config.UseServerModTime, "use-server-modtime", "", fs.Config.UseServerModTime, "Use server modified time instead of object metadata")

fs/sync/sync.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"path"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -36,6 +37,7 @@ type syncCopyMove struct {
 	deletersWg           sync.WaitGroup         // for delete before go routine
 	deleteFilesCh        chan fs.Object         // channel to receive deletes if delete before
 	trackRenames         bool                   // set if we should do server side renames
+	trackRenamesStrategy []string               // stratgies used for tracking renames
 	dstFilesMu           sync.Mutex             // protect dstFiles
 	dstFiles             map[string]fs.Object   // dst files, always filled
 	srcFiles             map[string]fs.Object   // src files, only used if deleteBefore
@@ -87,6 +89,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 		noCheckDest:          fs.Config.NoCheckDest,
 		deleteFilesCh:        make(chan fs.Object, fs.Config.Checkers),
 		trackRenames:         fs.Config.TrackRenames,
+		trackRenamesStrategy: strings.Split(fs.Config.TrackRenamesStrategy, ","),
 		commonHash:           fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(),
 		trackRenamesCh:       make(chan fs.Object, fs.Config.Checkers),
 	}
@@ -132,10 +135,16 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 			fs.Errorf(fdst, "Ignoring --track-renames as the destination does not support server-side move or copy")
 			s.trackRenames = false
 		}
-		if s.commonHash == hash.None {
+		if s.commonHash == hash.None && isUsingRenameStrategy("hash", s.trackRenamesStrategy) {
 			fs.Errorf(fdst, "Ignoring --track-renames as the source and destination do not have a common hash")
 			s.trackRenames = false
 		}
+
+		if fs.GetModifyWindow(fsrc, fdst) == fs.ModTimeNotSupported && isUsingRenameStrategy("modtime", s.trackRenamesStrategy) {
+			fs.Errorf(fdst, "Ignoring --track-renames as either the source or destination do not support modtime")
+			s.trackRenames = false
+		}
+
 		if s.deleteMode == fs.DeleteModeOff {
 			fs.Errorf(fdst, "Ignoring --track-renames as it doesn't work with copy or move, only sync")
 			s.trackRenames = false
@@ -560,12 +569,28 @@ func (s *syncCopyMove) srcParentDirCheck(entry fs.DirEntry) {
 	}
 }
 
-// renameHash makes a string with the size and the hash for rename detection
+// isUsingRenameStrategy checks if a strategy is included in a list of strategies
+func isUsingRenameStrategy(strategy string, strategies []string) bool {
+	for _, s := range strategies {
+		if s == strategy {
+			return true
+		}
+	}
+	return false
+}
+
+// renameID makes a string with the size and the other identifiers of the requested rename strategies
 //
 // it may return an empty string in which case no hash could be made
-func (s *syncCopyMove) renameHash(obj fs.Object) (hash string) {
+func (s *syncCopyMove) renameID(obj fs.Object, renameStrategy []string, precision time.Duration) string {
+	var builder strings.Builder
+
+	fmt.Fprintf(&builder, "%d", obj.Size())
+
+	if isUsingRenameStrategy("hash", renameStrategy) {
 		var err error
-	hash, err = obj.Hash(s.ctx, s.commonHash)
+		hash, err := obj.Hash(s.ctx, s.commonHash)
+
 		if err != nil {
 			fs.Debugf(obj, "Hash failed: %v", err)
 			return ""
@@ -573,7 +598,24 @@ func (s *syncCopyMove) renameHash(obj fs.Object) (hash string) {
 		if hash == "" {
 			return ""
 		}
-	return fmt.Sprintf("%d,%s", obj.Size(), hash)
+
+		fmt.Fprintf(&builder, ",%s", hash)
+	}
+
+	if isUsingRenameStrategy("modtime", renameStrategy) {
+		modTime := obj.ModTime(s.ctx)
+		divisor := precision.Nanoseconds() / 2
+		rounding := divisor / 2
+
+		if rounding > 0 {
+			rounding--
+		}
+
+		timeHash := (modTime.Unix()*time.Nanosecond.Nanoseconds() + int64(modTime.Nanosecond()) + rounding)
+		fmt.Fprintf(&builder, ",%d", timeHash)
+	}
+
+	return builder.String()
 }
 
 // pushRenameMap adds the object with hash to the rename map
@@ -626,10 +668,12 @@ func (s *syncCopyMove) makeRenameMap() {
 				// only create hash for dst fs.Object if its size could match
 				if _, found := possibleSizes[obj.Size()]; found {
 					tr := accounting.Stats(s.ctx).NewCheckingTransfer(obj)
-					hash := s.renameHash(obj)
+					hash := s.renameID(obj, s.trackRenamesStrategy, fs.GetModifyWindow(s.fsrc, s.fdst))
+
 					if hash != "" {
 						s.pushRenameMap(hash, obj)
 					}
+
 					tr.Done(nil)
 				}
 			}
@@ -643,7 +687,8 @@ func (s *syncCopyMove) makeRenameMap() {
 // possible, it returns true if the object was renamed.
 func (s *syncCopyMove) tryRename(src fs.Object) bool {
 	// Calculate the hash of the src object
-	hash := s.renameHash(src)
+	hash := s.renameID(src, s.trackRenamesStrategy, fs.GetModifyWindow(s.fsrc, s.fdst))
+
 	if hash == "" {
 		return false
 	}

fs/sync/sync_test.go

@@ -1127,6 +1127,98 @@ func TestSyncWithTrackRenames(t *testing.T) {
 	}
 }
 
+func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	fs.Config.TrackRenames = true
+	fs.Config.TrackRenamesStrategy = "hash,modtime"
+	defer func() {
+		fs.Config.TrackRenames = false
+		fs.Config.TrackRenamesStrategy = "hash"
+	}()
+
+	haveHash := r.Fremote.Hashes().Overlap(r.Flocal.Hashes()).GetOne() != hash.None
+	canTrackRenames := haveHash && operations.CanServerSideMove(r.Fremote)
+	t.Logf("Can track renames: %v", canTrackRenames)
+
+	f1 := r.WriteFile("potato", "Potato Content", t1)
+	f2 := r.WriteFile("yam", "Yam Content", t2)
+
+	accounting.GlobalStats().ResetCounters()
+	require.NoError(t, Sync(context.Background(), r.Fremote, r.Flocal, false))
+
+	fstest.CheckItems(t, r.Fremote, f1, f2)
+	fstest.CheckItems(t, r.Flocal, f1, f2)
+
+	// Now rename locally.
+	f2 = r.RenameFile(f2, "yaml")
+
+	accounting.GlobalStats().ResetCounters()
+	require.NoError(t, Sync(context.Background(), r.Fremote, r.Flocal, false))
+
+	fstest.CheckItems(t, r.Fremote, f1, f2)
+
+	// As currently there is no Fs interface providing number of chunks
+	// in a file, this test depends on the well-known names of test remotes.
+	remote := r.Fremote.Name()
+
+	// Union remote can Move but returns CantMove error.
+	moveAsCopyDelete := r.Fremote.Features().Move == nil || remote == "TestUnion"
+
+	chunker := strings.HasPrefix(remote, "TestChunker")
+	wrappedMoveAsCopyDelete := chunker && strings.HasSuffix(remote, "S3")
+
+	chunk3b := chunker && strings.Contains(remote, "Chunk3b")            // chunker with 3 byte chunks
+	chunk50b := chunker && strings.Contains(remote, "Chunk50b")          // chunker with 50 byte chunks
+	chunkDefault := chunker && !strings.Contains(remote, "ChunkerChunk") // default big chunk size
+	chunkBig := chunk50b || chunkDefault                                 // file is smaller than chunk size
+
+	// Verify number of checks for a toy 14 byte file.
+	// The order of cases matters!
+	var checks int
+	switch {
+	case canTrackRenames && chunk3b:
+		checks = 8 // chunker makes extra checks for each small chunk
+	case canTrackRenames && chunkBig:
+		checks = 4 // chunker makes 1 extra check for a single big chunk
+	case canTrackRenames && moveAsCopyDelete:
+		checks = 4 // 2 file checks + 1 move + 1 delete
+	case canTrackRenames:
+		checks = 3 // 2 file checks + 1 move
+	case !chunker:
+		checks = 2 // 2 file checks on a generic non-chunking remote
+	case chunk3b:
+		checks = 6 // chunker makes extra checks for each small chunk
+	case chunkBig && wrappedMoveAsCopyDelete:
+		checks = 4 // one more extra check because S3 emulates Move as Copy+Delete
+	case chunkBig:
+		checks = 3 // chunker makes 1 extra check for a single big chunk
+	default:
+		checks = -1 // skip verification for chunker with unknown chunk size
+	}
+	if checks != -1 { // "-1" allows remotes to bypass this check
+		assert.Equal(t, int64(checks), accounting.GlobalStats().GetChecks())
+	}
+
+	// Verify number of copy operations for a toy 14 byte file.
+	// The order of cases matters!
+	var copies int64
+	switch {
+	case canTrackRenames && moveAsCopyDelete:
+		copies = 1 // 1 copy
+	case canTrackRenames:
+		copies = 0 // 0 copy
+	case chunkBig && wrappedMoveAsCopyDelete:
+		copies = 2 // extra Copy because S3 emulates Move as Copy+Delete.
+	default:
+		copies = 1
+	}
+	if copies != -1 { // "-1" allows remotes to bypass this check
+		assert.Equal(t, copies, accounting.GlobalStats().GetTransfers())
+	}
+}
+
 func toyFileTransfers(r *fstest.Run) int64 {
 	remote := r.Fremote.Name()
 	transfers := 1