diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go
index 6fad8a4d4..cf1a5cd55 100644
--- a/fs/fshttp/http.go
+++ b/fs/fshttp/http.go
@@ -49,11 +49,13 @@ type timeoutConn struct {
 }
 
 // create a timeoutConn using the timeout
-func newTimeoutConn(conn net.Conn, timeout time.Duration) *timeoutConn {
-	return &timeoutConn{
+func newTimeoutConn(conn net.Conn, timeout time.Duration) (c *timeoutConn, err error) {
+	c = &timeoutConn{
 		Conn:    conn,
 		timeout: timeout,
 	}
+	err = c.nudgeDeadline()
+	return
 }
 
 // Nudge the deadline for an idle timeout on by c.timeout if non-zero
@@ -67,16 +69,14 @@ func (c *timeoutConn) nudgeDeadline() (err error) {
 
 // readOrWrite bytes doing idle timeouts
 func (c *timeoutConn) readOrWrite(f func([]byte) (int, error), b []byte) (n int, err error) {
-	err = c.nudgeDeadline()
-	if err != nil {
-		return n, err
-	}
 	n, err = f(b)
-	cerr := c.nudgeDeadline()
-	if err == nil && cerr != nil {
-		err = cerr
+	// Don't nudge if no bytes or an error
+	if n == 0 || err != nil {
+		return
 	}
-	return n, err
+	// Nudge the deadline on successful Read or Write
+	err = c.nudgeDeadline()
+	return
 }
 
 // Read bytes doing idle timeouts
diff --git a/fs/fshttp/http_new.go b/fs/fshttp/http_new.go
index 185f22e99..3726524af 100644
--- a/fs/fshttp/http_new.go
+++ b/fs/fshttp/http_new.go
@@ -20,7 +20,7 @@ func dialContextTimeout(ctx context.Context, network, address string, ci *fs.Con
 	if err != nil {
 		return c, err
 	}
-	return newTimeoutConn(c, ci.Timeout), nil
+	return newTimeoutConn(c, ci.Timeout)
 }
 
 // Initialise the http.Transport for go1.7+
diff --git a/fs/fshttp/http_old.go b/fs/fshttp/http_old.go
index d4c2bc68e..86ac3616d 100644
--- a/fs/fshttp/http_old.go
+++ b/fs/fshttp/http_old.go
@@ -18,7 +18,7 @@ func dialTimeout(network, address string, ci *fs.ConfigInfo) (net.Conn, error) {
 	if err != nil {
 		return c, err
 	}
-	return newTimeoutConn(c, ci.Timeout), nil
+	return newTimeoutConn(c, ci.Timeout)
 }
 
 // Initialise the http.Transport for pre go1.7