Implement --timeout and --conntimeout - fixes #54

NB dropbox still to do

README.md

@@ -133,14 +133,16 @@ This help.
 General options:
 
 ```
-      --bwlimit=0: Bandwidth limit in kBytes/s, or use suffix K|M|G
+      --bwlimit=0: Bandwidth limit in kBytes/s, or use suffix k|M|G
       --checkers=8: Number of checkers to run in parallel.
       --config="~/.rclone.conf": Config file.
+      --contimeout=1m0s: Connect timeout
   -n, --dry-run=false: Do a trial run with no permanent changes
       --log-file="": Log everything to this file
       --modify-window=1ns: Max time diff to be considered the same
   -q, --quiet=false: Print as little stuff as possible
       --stats=1m0s: Interval to print stats (0 to disable)
+      --timeout=5m0s: IO idle timeout
       --transfers=4: Number of file transfers to run in parallel.
   -v, --verbose=false: Print lots more stuff
   -V, --version=false: Print the version number

docs/content/docs.md

@@ -114,14 +114,16 @@ Enter an interactive configuration session.
 This help.
 
 ```
-      --bwlimit=0: Bandwidth limit in kBytes/s, or use suffix K|M|G
+      --bwlimit=0: Bandwidth limit in kBytes/s, or use suffix k|M|G
       --checkers=8: Number of checkers to run in parallel.
       --config="~/.rclone.conf": Config file.
+      --contimeout=1m0s: Connect timeout
   -n, --dry-run=false: Do a trial run with no permanent changes
       --log-file="": Log everything to this file
       --modify-window=1ns: Max time diff to be considered the same
   -q, --quiet=false: Print as little stuff as possible
       --stats=1m0s: Interval to print stats (0 to disable)
+      --timeout=5m0s: IO idle timeout
       --transfers=4: Number of file transfers to run in parallel.
   -v, --verbose=false: Print lots more stuff
   -V, --version=false: Print the version number

dropbox/dropbox.go

@@ -17,6 +17,20 @@ This is a JSON decode error - from Update / UploadByChunk
 - Caused by 500 error from dropbox
 - See https://github.com/stacktic/dropbox/issues/1
 - Possibly confusing dropbox with excess concurrency?
+
+FIXME implement timeouts - need to get "github.com/stacktic/dropbox"
+and hence "golang.org/x/oauth2" which uses DefaultTransport unless it
+is set in the context passed into .Client()
+
+func (db *Dropbox) client() *http.Client {
+	return db.config.Client(oauth2.NoContext, db.token)
+}
+
+// HTTPClient is the context key to use with golang.org/x/net/context's
+// WithValue function to associate an *http.Client value with a context.
+var HTTPClient ContextKey
+
+So pass in a context with HTTPClient set...
 */
 
 import (

fs/config.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"math"
+	"net/http"
 	"os"
 	"os/user"
 	"path"
@@ -16,6 +17,7 @@ import (
 	"time"
 
 	"github.com/Unknwon/goconfig"
+	"github.com/mreiferson/go-httpclient"
 	"github.com/ogier/pflag"
 )
 
@@ -36,14 +38,16 @@ var (
 	// Global config
 	Config = &ConfigInfo{}
 	// Flags
-	verbose      = pflag.BoolP("verbose", "v", false, "Print lots more stuff")
-	quiet        = pflag.BoolP("quiet", "q", false, "Print as little stuff as possible")
-	modifyWindow = pflag.DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same")
-	checkers     = pflag.IntP("checkers", "", 8, "Number of checkers to run in parallel.")
-	transfers    = pflag.IntP("transfers", "", 4, "Number of file transfers to run in parallel.")
-	configFile   = pflag.StringP("config", "", ConfigPath, "Config file.")
-	dryRun       = pflag.BoolP("dry-run", "n", false, "Do a trial run with no permanent changes")
-	bwLimit      SizeSuffix
+	verbose        = pflag.BoolP("verbose", "v", false, "Print lots more stuff")
+	quiet          = pflag.BoolP("quiet", "q", false, "Print as little stuff as possible")
+	modifyWindow   = pflag.DurationP("modify-window", "", time.Nanosecond, "Max time diff to be considered the same")
+	checkers       = pflag.IntP("checkers", "", 8, "Number of checkers to run in parallel.")
+	transfers      = pflag.IntP("transfers", "", 4, "Number of file transfers to run in parallel.")
+	configFile     = pflag.StringP("config", "", ConfigPath, "Config file.")
+	dryRun         = pflag.BoolP("dry-run", "n", false, "Do a trial run with no permanent changes")
+	connectTimeout = pflag.DurationP("contimeout", "", 60*time.Second, "Connect timeout")
+	timeout        = pflag.DurationP("timeout", "", 5*60*time.Second, "IO idle timeout")
+	bwLimit        SizeSuffix
 )
 
 func init() {
@@ -112,12 +116,48 @@ var _ pflag.Value = (*SizeSuffix)(nil)
 
 // Filesystem config options
 type ConfigInfo struct {
-	Verbose      bool
-	Quiet        bool
-	DryRun       bool
-	ModifyWindow time.Duration
-	Checkers     int
-	Transfers    int
+	Verbose        bool
+	Quiet          bool
+	DryRun         bool
+	ModifyWindow   time.Duration
+	Checkers       int
+	Transfers      int
+	ConnectTimeout time.Duration // Connect timeout
+	Timeout        time.Duration // Data channel timeout
+}
+
+// Transport returns an http.RoundTripper with the correct timeouts
+func (ci *ConfigInfo) Transport() http.RoundTripper {
+	return &httpclient.Transport{
+		Proxy:               http.ProxyFromEnvironment,
+		MaxIdleConnsPerHost: ci.Checkers + ci.Transfers + 1,
+
+		// ConnectTimeout, if non-zero, is the maximum amount of time a dial will wait for
+		// a connect to complete.
+		ConnectTimeout: ci.ConnectTimeout,
+
+		// ResponseHeaderTimeout, if non-zero, specifies the amount of
+		// time to wait for a server's response headers after fully
+		// writing the request (including its body, if any). This
+		// time does not include the time to read the response body.
+		ResponseHeaderTimeout: ci.Timeout,
+
+		// RequestTimeout, if non-zero, specifies the amount of time for the entire
+		// request to complete (including all of the above timeouts + entire response body).
+		// This should never be less than the sum total of the above two timeouts.
+		//RequestTimeout: NOT SET,
+
+		// ReadWriteTimeout, if non-zero, will set a deadline for every Read and
+		// Write operation on the request connection.
+		ReadWriteTimeout: ci.Timeout,
+	}
+}
+
+// Transport returns an http.Client with the correct timeouts
+func (ci *ConfigInfo) Client() *http.Client {
+	return &http.Client{
+		Transport: ci.Transport(),
+	}
 }
 
 // Find the config directory
@@ -152,6 +192,8 @@ func LoadConfig() {
 	Config.Checkers = *checkers
 	Config.Transfers = *transfers
 	Config.DryRun = *dryRun
+	Config.Timeout = *timeout
+	Config.ConnectTimeout = *connectTimeout
 
 	ConfigPath = *configFile
 

googleauth/googleauth.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
-	"net/http"
 
 	"code.google.com/p/goauth2/oauth"
 	"github.com/ncw/rclone/fs"
@@ -82,7 +81,7 @@ func (auth *Auth) newTransport(name string) (*oauth.Transport, error) {
 
 	t := &oauth.Transport{
 		Config:    config,
-		Transport: http.DefaultTransport,
+		Transport: fs.Config.Transport(),
 	}
 
 	return t, nil

s3/s3.go

@@ -184,6 +184,7 @@ func s3Connection(name string) (*s3.S3, error) {
 	}
 
 	c := s3.New(auth, region)
+	c.Client = fs.Config.Client()
 	return c, nil
 }
 

swift/swift.go

@@ -113,12 +113,15 @@ func swiftConnection(name string) (*swift.Connection, error) {
 		return nil, errors.New("auth not found")
 	}
 	c := &swift.Connection{
-		UserName:  userName,
-		ApiKey:    apiKey,
-		AuthUrl:   authUrl,
-		UserAgent: fs.UserAgent,
-		Tenant:    fs.ConfigFile.MustValue(name, "tenant"),
-		Region:    fs.ConfigFile.MustValue(name, "region"),
+		UserName:       userName,
+		ApiKey:         apiKey,
+		AuthUrl:        authUrl,
+		UserAgent:      fs.UserAgent,
+		Tenant:         fs.ConfigFile.MustValue(name, "tenant"),
+		Region:         fs.ConfigFile.MustValue(name, "region"),
+		ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
+		Timeout:        10 * fs.Config.Timeout,        // Use the timeouts in the transport
+		Transport:      fs.Config.Transport(),
 	}
 	err := c.Authenticate()
 	if err != nil {