mirror of
https://github.com/rclone/rclone.git
synced 2024-12-23 15:38:57 +01:00
sftp: close idle connections after --sftp-idle-timeout (1m by default)
This fixes a problem where sftp backends live on forever when using the rc and use more and more connections. Fixes #4883
This commit is contained in:
parent
a63e1f1383
commit
a3fcadddc8
@ -204,6 +204,17 @@ Fstat instead of Stat which is called on an already open file handle.
|
|||||||
It has been found that this helps with IBM Sterling SFTP servers which have
|
It has been found that this helps with IBM Sterling SFTP servers which have
|
||||||
"extractability" level set to 1 which means only 1 file can be opened at
|
"extractability" level set to 1 which means only 1 file can be opened at
|
||||||
any given time.
|
any given time.
|
||||||
|
`,
|
||||||
|
Advanced: true,
|
||||||
|
}, {
|
||||||
|
Name: "idle_timeout",
|
||||||
|
Default: fs.Duration(60 * time.Second),
|
||||||
|
Help: `Max time before closing idle connections
|
||||||
|
|
||||||
|
If no connections have been returned to the connection pool in the time
|
||||||
|
given, rclone will empty the connection pool.
|
||||||
|
|
||||||
|
Set to 0 to keep connections indefinitely.
|
||||||
`,
|
`,
|
||||||
Advanced: true,
|
Advanced: true,
|
||||||
}},
|
}},
|
||||||
@ -213,27 +224,28 @@ any given time.
|
|||||||
|
|
||||||
// Options defines the configuration for this backend
|
// Options defines the configuration for this backend
|
||||||
type Options struct {
|
type Options struct {
|
||||||
Host string `config:"host"`
|
Host string `config:"host"`
|
||||||
User string `config:"user"`
|
User string `config:"user"`
|
||||||
Port string `config:"port"`
|
Port string `config:"port"`
|
||||||
Pass string `config:"pass"`
|
Pass string `config:"pass"`
|
||||||
KeyPem string `config:"key_pem"`
|
KeyPem string `config:"key_pem"`
|
||||||
KeyFile string `config:"key_file"`
|
KeyFile string `config:"key_file"`
|
||||||
KeyFilePass string `config:"key_file_pass"`
|
KeyFilePass string `config:"key_file_pass"`
|
||||||
PubKeyFile string `config:"pubkey_file"`
|
PubKeyFile string `config:"pubkey_file"`
|
||||||
KnownHostsFile string `config:"known_hosts_file"`
|
KnownHostsFile string `config:"known_hosts_file"`
|
||||||
KeyUseAgent bool `config:"key_use_agent"`
|
KeyUseAgent bool `config:"key_use_agent"`
|
||||||
UseInsecureCipher bool `config:"use_insecure_cipher"`
|
UseInsecureCipher bool `config:"use_insecure_cipher"`
|
||||||
DisableHashCheck bool `config:"disable_hashcheck"`
|
DisableHashCheck bool `config:"disable_hashcheck"`
|
||||||
AskPassword bool `config:"ask_password"`
|
AskPassword bool `config:"ask_password"`
|
||||||
PathOverride string `config:"path_override"`
|
PathOverride string `config:"path_override"`
|
||||||
SetModTime bool `config:"set_modtime"`
|
SetModTime bool `config:"set_modtime"`
|
||||||
Md5sumCommand string `config:"md5sum_command"`
|
Md5sumCommand string `config:"md5sum_command"`
|
||||||
Sha1sumCommand string `config:"sha1sum_command"`
|
Sha1sumCommand string `config:"sha1sum_command"`
|
||||||
SkipLinks bool `config:"skip_links"`
|
SkipLinks bool `config:"skip_links"`
|
||||||
Subsystem string `config:"subsystem"`
|
Subsystem string `config:"subsystem"`
|
||||||
ServerCommand string `config:"server_command"`
|
ServerCommand string `config:"server_command"`
|
||||||
UseFstat bool `config:"use_fstat"`
|
UseFstat bool `config:"use_fstat"`
|
||||||
|
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fs stores the interface to the remote SFTP files
|
// Fs stores the interface to the remote SFTP files
|
||||||
@ -251,7 +263,8 @@ type Fs struct {
|
|||||||
cachedHashes *hash.Set
|
cachedHashes *hash.Set
|
||||||
poolMu sync.Mutex
|
poolMu sync.Mutex
|
||||||
pool []*conn
|
pool []*conn
|
||||||
pacer *fs.Pacer // pacer for operations
|
drain *time.Timer // used to drain the pool when we stop using the connections
|
||||||
|
pacer *fs.Pacer // pacer for operations
|
||||||
savedpswd string
|
savedpswd string
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -428,6 +441,9 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
|
|||||||
}
|
}
|
||||||
f.poolMu.Lock()
|
f.poolMu.Lock()
|
||||||
f.pool = append(f.pool, c)
|
f.pool = append(f.pool, c)
|
||||||
|
if f.opt.IdleTimeout > 0 {
|
||||||
|
f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
|
||||||
|
}
|
||||||
f.poolMu.Unlock()
|
f.poolMu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -435,6 +451,12 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
|
|||||||
func (f *Fs) drainPool(ctx context.Context) (err error) {
|
func (f *Fs) drainPool(ctx context.Context) (err error) {
|
||||||
f.poolMu.Lock()
|
f.poolMu.Lock()
|
||||||
defer f.poolMu.Unlock()
|
defer f.poolMu.Unlock()
|
||||||
|
if f.opt.IdleTimeout > 0 {
|
||||||
|
f.drain.Stop()
|
||||||
|
}
|
||||||
|
if len(f.pool) != 0 {
|
||||||
|
fs.Debugf(f, "closing %d unused connections", len(f.pool))
|
||||||
|
}
|
||||||
for i, c := range f.pool {
|
for i, c := range f.pool {
|
||||||
if cErr := c.closed(); cErr == nil {
|
if cErr := c.closed(); cErr == nil {
|
||||||
cErr = c.close()
|
cErr = c.close()
|
||||||
@ -667,6 +689,10 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
|
|||||||
f.mkdirLock = newStringLock()
|
f.mkdirLock = newStringLock()
|
||||||
f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
|
f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
|
||||||
f.savedpswd = ""
|
f.savedpswd = ""
|
||||||
|
// set the pool drainer timer going
|
||||||
|
if f.opt.IdleTimeout > 0 {
|
||||||
|
f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
|
||||||
|
}
|
||||||
|
|
||||||
f.features = (&fs.Features{
|
f.features = (&fs.Features{
|
||||||
CanHaveEmptyDirectories: true,
|
CanHaveEmptyDirectories: true,
|
||||||
|
Loading…
Reference in New Issue
Block a user