extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-25 18:04:55 +01:00
Code Issues Packages Projects Releases Wiki Activity

sftp: stop uploads re-using the same ssh connection to improve performance

Browse Source 
Before this change we released the ssh connection back to the pool
before the upload was finished.

This meant that uploads were re-using the same ssh connection which
reduces throughput.

This releases the ssh connection back to the pool only after the
upload has finished, or on error state.

See: https://forum.rclone.org/t/sftp-backend-opens-less-connection-than-expected/40245
This commit is contained in:
Nick Craig-Wood 2023-07-24 23:23:46 +01:00
parent aaadb48d48
commit 3ed4a2e963
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
backend/sftp/sftp.go
View File

@ -1992,9 +1992,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if err != nil { if err != nil {
return fmt.Errorf("Update: %w", err) return fmt.Errorf("Update: %w", err)
} }
// Hang on to the connection for the whole upload so it doesn't get re-used while we are uploading
file, err := c.sftpClient.OpenFile(o.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC) file, err := c.sftpClient.OpenFile(o.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
o.fs.putSftpConnection(&c, err)
if err != nil { if err != nil {
o.fs.putSftpConnection(&c, err)
return fmt.Errorf("Update Create failed: %w", err) return fmt.Errorf("Update Create failed: %w", err)
} }
// remove the file if upload failed // remove the file if upload failed
@ -2014,14 +2015,18 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
_, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()}) _, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()})
if err != nil { if err != nil {
o.fs.putSftpConnection(&c, err)
remove() remove()
return fmt.Errorf("Update ReadFrom failed: %w", err) return fmt.Errorf("Update ReadFrom failed: %w", err)
} }
err = file.Close() err = file.Close()
if err != nil { if err != nil {
o.fs.putSftpConnection(&c, err)
remove() remove()
return fmt.Errorf("Update Close failed: %w", err) return fmt.Errorf("Update Close failed: %w", err)
} }
// Release connection only when upload has finished so we don't upload multiple files on the same connection
o.fs.putSftpConnection(&c, err)
// Set the mod time - this stats the object if o.fs.opt.SetModTime == true // Set the mod time - this stats the object if o.fs.opt.SetModTime == true
err = o.SetModTime(ctx, src.ModTime(ctx)) err = o.SetModTime(ctx, src.ModTime(ctx))