Implement --low-level-retries flag - fixes #266

docs/content/docs.md

@@ -363,6 +363,22 @@ Log all of rclone's output to FILE.  This is not active by default.
 This can be useful for tracking down problems with syncs in
 combination with the `-v` flag.
 
+### --low-level-retries NUMBER ###
+
+This controls the number of low level retries rclone does.
+
+A low level retry is used to retry a failing operation - typically one
+HTTP request.  This might be uploading a chunk of a big file for
+example.  You will see low level retries in the log with the `-v`
+flag.
+
+This shouldn't need to be changed from the default in normal
+operations, however if you get a lot of low level retries you may wish
+to reduce the value so rclone moves on to a high level retry (see the
+`--retries` flag) quicker.
+
+Disable low level retries with `--low-level-retries 1`.
+
 ### --modify-window=TIME ###
 
 When checking whether a file has been modified, this is the maximum

fs/config.go

@@ -80,6 +80,7 @@ var (
 	deleteBefore    = pflag.BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering")
 	deleteDuring    = pflag.BoolP("delete-during", "", false, "When synchronizing, delete files during transfer (default)")
 	deleteAfter     = pflag.BoolP("delete-after", "", false, "When synchronizing, delete files on destination after transfering")
+	lowLevelRetries = pflag.IntP("low-level-retries", "", 10, "Number of low level retries to do.")
 	bwLimit         SizeSuffix
 
 	// Key to use for password en/decryption.
@@ -197,6 +198,7 @@ type ConfigInfo struct {
 	DeleteBefore       bool // Delete before checking
 	DeleteDuring       bool // Delete during checking/transfer
 	DeleteAfter        bool // Delete after successful transfer.
+	LowLevelRetries    int
 }
 
 // Transport returns an http.RoundTripper with the correct timeouts
@@ -285,6 +287,7 @@ func LoadConfig() {
 	Config.DumpHeaders = *dumpHeaders
 	Config.DumpBodies = *dumpBodies
 	Config.InsecureSkipVerify = *skipVerify
+	Config.LowLevelRetries = *lowLevelRetries
 
 	ConfigPath = *configFile
 

fs/operations.go

@@ -185,7 +185,7 @@ func removeFailedCopy(dst Object) bool {
 // call Copy() with dst nil on a pre-existing file then some filing
 // systems (eg Drive) may duplicate the file.
 func Copy(f Fs, dst, src Object) {
-	const maxTries = 10
+	maxTries := Config.LowLevelRetries
 	tries := 0
 	doUpdate := dst != nil
 	var err, inErr error
@@ -231,7 +231,7 @@ tryAgain:
 	// Retry if err returned a retry error
 	if r, ok := err.(Retry); ok && r.Retry() && tries < maxTries {
 		tries++
-		Log(src, "Received error: %v - retrying %d/%d", err, tries, maxTries)
+		Log(src, "Received error: %v - low level retry %d/%d", err, tries, maxTries)
 		if removeFailedCopy(dst) {
 			// If we removed dst, then nil it out and note we are not updating
 			dst = nil

fs/operations_test.go

@@ -45,6 +45,7 @@ var (
 	DumpHeaders     = flag.Bool("dump-headers", false, "Set to dump headers (needs -verbose)")
 	DumpBodies      = flag.Bool("dump-bodies", false, "Set to dump bodies (needs -verbose)")
 	Individual      = flag.Bool("individual", false, "Make individual bucket/container/directory for each test - much slower")
+	LowLevelRetries = flag.Int("low-level-retries", 10, "Number of low level retries")
 )
 
 // Some times used in the tests
@@ -103,7 +104,7 @@ func newRun() *Run {
 	fs.Config.Quiet = !*Verbose
 	fs.Config.DumpHeaders = *DumpHeaders
 	fs.Config.DumpBodies = *DumpBodies
-
+	fs.Config.LowLevelRetries = *LowLevelRetries
 	var err error
 	r.fremote, r.cleanRemote, err = fstest.RandomRemote(*RemoteName, *SubDir)
 	if err != nil {

pacer/pacer.go

@@ -61,7 +61,7 @@ func New() *Pacer {
 		minSleep:      10 * time.Millisecond,
 		maxSleep:      2 * time.Second,
 		decayConstant: 2,
-		retries:       10,
+		retries:       fs.Config.LowLevelRetries,
 		pacer:         make(chan struct{}, 1),
 	}
 	p.sleepTime = p.minSleep
@@ -231,7 +231,7 @@ func (p *Pacer) acdPacer(retry bool) {
 		if p.sleepTime < p.minSleep {
 			p.sleepTime = p.minSleep
 		}
-		fs.Debug("pacer", "Rate limited, sleeping for %v (%d retries)", p.sleepTime, consecutiveRetries)
+		fs.Debug("pacer", "Rate limited, sleeping for %v (%d consecutive low level retries)", p.sleepTime, consecutiveRetries)
 	}
 }
 
@@ -256,13 +256,14 @@ func (p *Pacer) endCall(retry bool) {
 // call implements Call but with settable retries
 func (p *Pacer) call(fn Paced, retries int) (err error) {
 	var retry bool
-	for i := 0; i < retries; i++ {
+	for i := 1; i <= retries; i++ {
 		p.beginCall()
 		retry, err = fn()
 		p.endCall(retry)
 		if !retry {
 			break
 		}
+		fs.Debug("pacer", "low level retry %d/%d", i, retries)
 	}
 	if retry {
 		err = fs.RetryError(err)