From e77532852307b4f86e8ed09038c4dfc717173db4 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 6 Jan 2021 12:19:23 +0000 Subject: [PATCH] ftp,sftp: Make --tpslimit apply - fixes #4906 --- backend/ftp/ftp.go | 2 ++ backend/sftp/sftp.go | 2 ++ docs/content/docs.md | 15 ++++++++++++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index a27fd267b..387ef1f78 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -15,6 +15,7 @@ import ( "github.com/jlaffaye/ftp" "github.com/pkg/errors" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" @@ -258,6 +259,7 @@ func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error if f.opt.Concurrency > 0 { f.tokens.Get() } + accounting.LimitTPS(ctx) f.poolMu.Lock() if len(f.pool) > 0 { c = f.pool[0] diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 60be731b2..e7c93c775 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" "github.com/pkg/sftp" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" @@ -349,6 +350,7 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C // Get an SFTP connection from the pool, or open a new one func (f *Fs) getSftpConnection(ctx context.Context) (c *conn, err error) { + accounting.LimitTPS(ctx) f.poolMu.Lock() for len(f.pool) > 0 { c = f.pool[0] diff --git a/docs/content/docs.md b/docs/content/docs.md index 4590a26c4..2de1c664f 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -1297,10 +1297,15 @@ facility is `DAEMON`. ### --tpslimit float ### -Limit HTTP transactions per second to this. Default is 0 which is used -to mean unlimited transactions per second. +Limit transactions per second to this number. Default is 0 which is +used to mean unlimited transactions per second. -For example to limit rclone to 10 HTTP transactions per second use +A transaction is roughly defined as an API call; its exact meaning +will depend on the backend. For HTTP based backends it is an HTTP +PUT/GET/POST/etc and its response. For FTP/SFTP it is a round trip +transaction over TCP. + +For example to limit rclone to 10 transactions per second use `--tpslimit 10`, or to 1 transaction every 2 seconds use `--tpslimit 0.5`. @@ -1311,6 +1316,10 @@ banned or rate limited). This can be very useful for `rclone mount` to control the behaviour of applications using it. +This limit applies to all HTTP based backends and to the FTP and SFTP +backends. It does not apply to the local backend or the Tardigrade +backend. + See also `--tpslimit-burst`. ### --tpslimit-burst int ###