diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.go index c3a2e1e57..54a417cb4 100644 --- a/cmd/bisync/cmd.go +++ b/cmd/bisync/cmd.go @@ -46,6 +46,7 @@ type Options struct { IgnoreListingChecksum bool Resilient bool TestFn TestFunc // test-only option, for mocking errors + Retries int } // Default values @@ -103,6 +104,7 @@ func (x *CheckSyncMode) Type() string { var Opt Options func init() { + Opt.Retries = 3 cmd.Root.AddCommand(commandDefinition) cmdFlags := commandDefinition.Flags() flags.BoolVarP(cmdFlags, &Opt.Resync, "resync", "1", Opt.Resync, "Performs the resync run. Path1 files may overwrite Path2 versions. Consider using --verbose or --dry-run first.", "") @@ -118,6 +120,7 @@ func init() { flags.BoolVarP(cmdFlags, &Opt.NoCleanup, "no-cleanup", "", Opt.NoCleanup, "Retain working files (useful for troubleshooting and testing).", "") flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "") flags.BoolVarP(cmdFlags, &Opt.Resilient, "resilient", "", Opt.Resilient, "Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!", "") + flags.IntVarP(cmdFlags, &Opt.Retries, "retries", "", Opt.Retries, "Retry operations this many times if they fail", "") } // bisync command definition diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go index 68fb538c3..770dac750 100644 --- a/cmd/bisync/deltas.go +++ b/cmd/bisync/deltas.go @@ -427,6 +427,10 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change changes1 = true b.indent("Path2", "Path1", "Do queued copies to") results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1") + + // retries, if any + results2to1, err = b.retryFastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1", results2to1, err) + if err != nil { return } @@ -439,6 +443,10 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change changes2 = true b.indent("Path1", "Path2", "Do queued copies to") results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2") + + // retries, if any + results1to2, err = b.retryFastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2", results1to2, err) + if err != nil { return } diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.go index f08a82f89..1c4a56b8e 100644 --- a/cmd/bisync/queue.go +++ b/cmd/bisync/queue.go @@ -14,6 +14,7 @@ import ( "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/sync" + "github.com/rclone/rclone/lib/terminal" ) // Results represents a pair of synced files, as reported by the LoggerFn @@ -182,6 +183,16 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib. return getResults, err } +func (b *bisyncRun) retryFastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string, results []Results, err error) ([]Results, error) { + if err != nil && b.opt.Resilient && b.opt.Retries > 1 { + for tries := 1; tries <= b.opt.Retries; tries++ { + fs.Logf(queueName, Color(terminal.YellowFg, "Received error: %v - retrying as --resilient is set. Retry %d/%d"), err, tries, b.opt.Retries) + results, err = b.fastCopy(ctx, fsrc, fdst, files, queueName) + } + } + return results, err +} + func (b *bisyncRun) resyncDir(ctx context.Context, fsrc, fdst fs.Fs) ([]Results, error) { ignoreListingChecksum = b.opt.IgnoreListingChecksum logger.LoggerFn = WriteResults