sftp: implement --sftp-copy-is-hardlink to server side copy as hardlink

If the server does not support hardlinks then it falls back to normal
copy.

See: https://forum.rclone.org/t/sftp-remote-server-side-copy/41867
This commit is contained in:
Nick Craig-Wood
2023-09-21 14:55:48 +01:00
parent c190b9b14f
commit c7a2719fac
3 changed files with 120 additions and 56 deletions

View File

@ -449,6 +449,26 @@ Example:
myUser:myPass@localhost:9005
`,
Advanced: true,
}, {
Name: "copy_is_hardlink",
Default: false,
Help: `Set to enable server side copies using hardlinks.
The SFTP protocol does not define a copy command so normally server
side copies are not allowed with the sftp backend.
However the SFTP protocol does support hardlinking, and if you enable
this flag then the sftp backend will support server side copies. These
will be implemented by doing a hardlink from the source to the
destination.
Not all sftp servers support this.
Note that hardlinking two files together will use no additional space
as the source and the destination will be the same file.
This feature may be useful backups made with --copy-dest.`,
Advanced: true,
}},
}
fs.Register(fsi)
@ -490,6 +510,7 @@ type Options struct {
HostKeyAlgorithms fs.SpaceSepList `config:"host_key_algorithms"`
SSH fs.SpaceSepList `config:"ssh"`
SocksProxy string `config:"socks_proxy"`
CopyIsHardlink bool `config:"copy_is_hardlink"`
}
// Fs stores the interface to the remote SFTP files
@ -1049,6 +1070,10 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
SlowHash: true,
PartialUploads: true,
}).Fill(ctx, f)
if !opt.CopyIsHardlink {
// Disable server side copy unless --sftp-copy-is-hardlink is set
f.features.Copy = nil
}
// Make a connection and pool it to return errors early
c, err := f.getSftpConnection(ctx)
if err != nil {
@ -1401,6 +1426,43 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
return dstObj, nil
}
// Copy server side copies a remote sftp file object using hardlinks
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
if !f.opt.CopyIsHardlink {
return nil, fs.ErrorCantCopy
}
srcObj, ok := src.(*Object)
if !ok {
fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
err := f.mkParentDir(ctx, remote)
if err != nil {
return nil, fmt.Errorf("Copy mkParentDir failed: %w", err)
}
c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, fmt.Errorf("Copy: %w", err)
}
srcPath, dstPath := srcObj.path(), path.Join(f.absRoot, remote)
err = c.sftpClient.Link(srcPath, dstPath)
f.putSftpConnection(&c, err)
if err != nil {
if sftpErr, ok := err.(*sftp.StatusError); ok {
if sftpErr.FxCode() == sftp.ErrSSHFxOpUnsupported {
// Remote doesn't support Link
return nil, fs.ErrorCantCopy
}
}
return nil, fmt.Errorf("Copy failed: %w", err)
}
dstObj, err := f.NewObject(ctx, remote)
if err != nil {
return nil, fmt.Errorf("Copy NewObject failed: %w", err)
}
return dstObj, nil
}
// DirMove moves src, srcRemote to this remote at dstRemote
// using server-side move operations.
//
@ -2120,6 +2182,7 @@ var (
_ fs.Fs = &Fs{}
_ fs.PutStreamer = &Fs{}
_ fs.Mover = &Fs{}
_ fs.Copier = &Fs{}
_ fs.DirMover = &Fs{}
_ fs.Abouter = &Fs{}
_ fs.Shutdowner = &Fs{}