From 518d39815c2c6f9b5199a5d26a28ae260a1c97c2 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 15 May 2020 11:39:07 +0100 Subject: [PATCH] sync,copy,move: add --check-first to do all checking before starting transfers See: https://forum.rclone.org/t/rclone-sync-doing-transfer-and-checking-in-paralel/16352/ --- docs/content/docs.md | 14 ++++++++++++++ fs/config.go | 1 + fs/config/configflags/configflags.go | 1 + fs/sync/sync.go | 21 +++++++++++++++++---- fs/sync/sync_test.go | 17 +++++++++++++++++ 5 files changed, 50 insertions(+), 4 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index 8585fff74..a86b7de2d 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -429,6 +429,20 @@ Set to 0 to disable the buffering for the minimum memory usage. Note that the memory allocation of the buffers is influenced by the [--use-mmap](#use-mmap) flag. +### --check-first ### + +If this flag is set then in a `sync`, `copy` or `move`, rclone will do +all the checks to see whether files need to be transferred before +doing any of the transfers. Normally rclone would start running +transfers as soon as possible. + +This flag can be useful on IO limited systems where transfers +interfere with checking. + +Using this flag can use more memory as it effectively sets +`--max-backlog` to infinite. This means that all the info on the +objects to transfer is held in memory before the transfers start. + ### --checkers=N ### The number of checkers to run in parallel. Checkers do the equality diff --git a/fs/config.go b/fs/config.go index ba6bb2b1c..233e9e64c 100644 --- a/fs/config.go +++ b/fs/config.go @@ -69,6 +69,7 @@ type ConfigInfo struct { IgnoreChecksum bool IgnoreCaseSync bool NoTraverse bool + CheckFirst bool NoCheckDest bool NoUnicodeNormalization bool NoUpdateModTime bool diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index b66cd421b..42ac28db1 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -74,6 +74,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.") flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing") flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.") + flags.BoolVarP(flagSet, &fs.Config.CheckFirst, "check-first", "", fs.Config.CheckFirst, "Do all the checks before starting transfers.") flags.BoolVarP(flagSet, &fs.Config.NoCheckDest, "no-check-dest", "", fs.Config.NoCheckDest, "Don't check the destination, copy regardless.") flags.BoolVarP(flagSet, &fs.Config.NoUnicodeNormalization, "no-unicode-normalization", "", fs.Config.NoUnicodeNormalization, "Don't normalize unicode characters in filenames.") flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.") diff --git a/fs/sync/sync.go b/fs/sync/sync.go index d392aabed..bc579fee6 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -67,6 +67,7 @@ type syncCopyMove struct { renameCheck []fs.Object // accumulate files to check for rename here compareCopyDest fs.Fs // place to check for files to server side copy backupDir fs.Fs // place to store overwrites/deletes + checkFirst bool // if set run all the checkers before starting transfers } type trackRenamesStrategy byte @@ -108,17 +109,23 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete trackRenames: fs.Config.TrackRenames, commonHash: fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(), trackRenamesCh: make(chan fs.Object, fs.Config.Checkers), + checkFirst: fs.Config.CheckFirst, + } + backlog := fs.Config.MaxBacklog + if s.checkFirst { + fs.Infof(s.fdst, "Running all checks before starting transfers") + backlog = -1 } var err error - s.toBeChecked, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetCheckQueue, fs.Config.MaxBacklog) + s.toBeChecked, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetCheckQueue, backlog) if err != nil { return nil, err } - s.toBeUploaded, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetTransferQueue, fs.Config.MaxBacklog) + s.toBeUploaded, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetTransferQueue, backlog) if err != nil { return nil, err } - s.toBeRenamed, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetRenameQueue, fs.Config.MaxBacklog) + s.toBeRenamed, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetRenameQueue, backlog) if err != nil { return nil, err } @@ -776,7 +783,9 @@ func (s *syncCopyMove) run() error { // Start background checking and transferring pipeline s.startCheckers() s.startRenamers() - s.startTransfers() + if !s.checkFirst { + s.startTransfers() + } s.startDeleters() s.dstFiles = make(map[string]fs.Object) @@ -811,6 +820,10 @@ func (s *syncCopyMove) run() error { // Stop background checking and transferring pipeline s.stopCheckers() + if s.checkFirst { + fs.Infof(s.fdst, "Checks finished, now starting transfers") + s.startTransfers() + } s.stopRenamers() s.stopTransfers() s.stopDeleters() diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index ceb9f5dfc..8a74c15b1 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -97,6 +97,23 @@ func TestCopyNoTraverse(t *testing.T) { fstest.CheckItems(t, r.Fremote, file1) } +// Now with --check-first +func TestCopyCheckFirst(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + fs.Config.CheckFirst = true + defer func() { fs.Config.CheckFirst = false }() + + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + + err := CopyDir(context.Background(), r.Fremote, r.Flocal, false) + require.NoError(t, err) + + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) +} + // Now with --no-traverse func TestSyncNoTraverse(t *testing.T) { r := fstest.NewRun(t)