sync,copy,move: add --check-first to do all checking before starting transfers

See: https://forum.rclone.org/t/rclone-sync-doing-transfer-and-checking-in-paralel/16352/

docs/content/docs.md

@@ -429,6 +429,20 @@ Set to 0 to disable the buffering for the minimum memory usage.
 Note that the memory allocation of the buffers is influenced by the
 [--use-mmap](#use-mmap) flag.
 
+### --check-first ###
+
+If this flag is set then in a `sync`, `copy` or `move`, rclone will do
+all the checks to see whether files need to be transferred before
+doing any of the transfers. Normally rclone would start running
+transfers as soon as possible.
+
+This flag can be useful on IO limited systems where transfers
+interfere with checking.
+
+Using this flag can use more memory as it effectively sets
+`--max-backlog` to infinite. This means that all the info on the
+objects to transfer is held in memory before the transfers start.
+
 ### --checkers=N ###
 
 The number of checkers to run in parallel.  Checkers do the equality

fs/config.go

@@ -69,6 +69,7 @@ type ConfigInfo struct {
 	IgnoreChecksum         bool
 	IgnoreCaseSync         bool
 	NoTraverse             bool
+	CheckFirst             bool
 	NoCheckDest            bool
 	NoUnicodeNormalization bool
 	NoUpdateModTime        bool

fs/config/configflags/configflags.go

@@ -74,6 +74,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.")
 	flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing")
 	flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.")
+	flags.BoolVarP(flagSet, &fs.Config.CheckFirst, "check-first", "", fs.Config.CheckFirst, "Do all the checks before starting transfers.")
 	flags.BoolVarP(flagSet, &fs.Config.NoCheckDest, "no-check-dest", "", fs.Config.NoCheckDest, "Don't check the destination, copy regardless.")
 	flags.BoolVarP(flagSet, &fs.Config.NoUnicodeNormalization, "no-unicode-normalization", "", fs.Config.NoUnicodeNormalization, "Don't normalize unicode characters in filenames.")
 	flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.")

fs/sync/sync.go

@@ -67,6 +67,7 @@ type syncCopyMove struct {
 	renameCheck            []fs.Object            // accumulate files to check for rename here
 	compareCopyDest        fs.Fs                  // place to check for files to server side copy
 	backupDir              fs.Fs                  // place to store overwrites/deletes
+	checkFirst             bool                   // if set run all the checkers before starting transfers
 }
 
 type trackRenamesStrategy byte
@@ -108,17 +109,23 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 		trackRenames:           fs.Config.TrackRenames,
 		commonHash:             fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(),
 		trackRenamesCh:         make(chan fs.Object, fs.Config.Checkers),
+		checkFirst:             fs.Config.CheckFirst,
+	}
+	backlog := fs.Config.MaxBacklog
+	if s.checkFirst {
+		fs.Infof(s.fdst, "Running all checks before starting transfers")
+		backlog = -1
 	}
 	var err error
-	s.toBeChecked, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetCheckQueue, fs.Config.MaxBacklog)
+	s.toBeChecked, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetCheckQueue, backlog)
 	if err != nil {
 		return nil, err
 	}
-	s.toBeUploaded, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetTransferQueue, fs.Config.MaxBacklog)
+	s.toBeUploaded, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetTransferQueue, backlog)
 	if err != nil {
 		return nil, err
 	}
-	s.toBeRenamed, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetRenameQueue, fs.Config.MaxBacklog)
+	s.toBeRenamed, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetRenameQueue, backlog)
 	if err != nil {
 		return nil, err
 	}
@@ -776,7 +783,9 @@ func (s *syncCopyMove) run() error {
 	// Start background checking and transferring pipeline
 	s.startCheckers()
 	s.startRenamers()
+	if !s.checkFirst {
 		s.startTransfers()
+	}
 	s.startDeleters()
 	s.dstFiles = make(map[string]fs.Object)
 
@@ -811,6 +820,10 @@ func (s *syncCopyMove) run() error {
 
 	// Stop background checking and transferring pipeline
 	s.stopCheckers()
+	if s.checkFirst {
+		fs.Infof(s.fdst, "Checks finished, now starting transfers")
+		s.startTransfers()
+	}
 	s.stopRenamers()
 	s.stopTransfers()
 	s.stopDeleters()

fs/sync/sync_test.go

@@ -97,6 +97,23 @@ func TestCopyNoTraverse(t *testing.T) {
 	fstest.CheckItems(t, r.Fremote, file1)
 }
 
+// Now with --check-first
+func TestCopyCheckFirst(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	fs.Config.CheckFirst = true
+	defer func() { fs.Config.CheckFirst = false }()
+
+	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
+
+	err := CopyDir(context.Background(), r.Fremote, r.Flocal, false)
+	require.NoError(t, err)
+
+	fstest.CheckItems(t, r.Flocal, file1)
+	fstest.CheckItems(t, r.Fremote, file1)
+}
+
 // Now with --no-traverse
 func TestSyncNoTraverse(t *testing.T) {
 	r := fstest.NewRun(t)