From 773620ca0b068b0459a622c4159d96dc7a679dd4 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Thu, 29 Aug 2024 23:41:00 +0200 Subject: [PATCH] sftp: add symlink support - fixes #5011 Add new flag (--sftp-links) for backend sftp to recreate symlink from source to destination. --- backend/sftp/sftp.go | 176 ++++++++++++++++++++++++++++++++++++++----- docs/content/sftp.md | 47 ++++++++---- fs/features.go | 1 + fs/fs.go | 1 + 4 files changed, 189 insertions(+), 36 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index e47e47db2..c878a7df8 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -178,13 +178,13 @@ E.g. if shared folders can be found in directories representing volumes: E.g. if home directory can be found in a shared folder called "home": rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory - + To specify only the path to the SFTP remote's root, and allow rclone to add any relative subpaths automatically (including unwrapping/decrypting remotes as necessary), add the '@' character to the beginning of the path. E.g. the first example above could be rewritten as: rclone sync /home/local/directory remote:/directory --sftp-path-override @/volume2 - + Note that when using this method with Synology "home" folders, the full "/homes/USER" path should be specified instead of "/home". E.g. the second example above should be rewritten as: @@ -232,6 +232,19 @@ E.g. the second example above should be rewritten as: Help: "Set to skip any symlinks and any other non regular files.", Advanced: true, }, { + Name: "links", + Default: false, + Help: `Copy symlinks instead of following them. + +This permit to recreate the same symlink structure on the destination. + +Only works between two remotes with symlink support. [Currently only supported between two SFTP remotes]. + +Symlink is validate if target file on destination and on source have same size and same hash. [Except if target is a directory, size and hash are not checked]. +`, + Advanced: true, + }, { + Name: "subsystem", Default: "sftp", Help: "Specifies the SSH2 subsystem on the remote host.", @@ -243,7 +256,7 @@ E.g. the second example above should be rewritten as: The subsystem option is ignored when server_command is defined. -If adding server_command to the configuration file please note that +If adding server_command to the configuration file please note that it should not be enclosed in quotes, since that will make rclone fail. A working example is: @@ -469,7 +482,7 @@ connection for every hash it calculates. Name: "socks_proxy", Default: "", Help: `Socks 5 proxy host. - + Supports the format user:pass@host:port, user@host:port, host:port. Example: @@ -523,6 +536,7 @@ type Options struct { Md5sumCommand string `config:"md5sum_command"` Sha1sumCommand string `config:"sha1sum_command"` SkipLinks bool `config:"skip_links"` + TranslateSymlinks bool `config:"links"` Subsystem string `config:"subsystem"` ServerCommand string `config:"server_command"` UseFstat bool `config:"use_fstat"` @@ -568,13 +582,15 @@ type Fs struct { // Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading) type Object struct { - fs *Fs - remote string - size int64 // size of the object - modTime uint32 // modification time of the object as unix time - mode os.FileMode // mode bits from the file - md5sum *string // Cached MD5 checksum - sha1sum *string // Cached SHA1 checksum + fs *Fs + remote string + size int64 // size of the object + modTime uint32 // modification time of the object as unix time + mode os.FileMode // mode bits from the file + md5sum *string // Cached MD5 checksum + sha1sum *string // Cached SHA1 checksum + linkTarget string // If object isSymlink, this is the target + linkTargetIsDir bool // If object isSymlink, this is true if the target is a directory } // conn encapsulates an ssh client and corresponding sftp client @@ -852,6 +868,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e if len(opt.SSH) != 0 && ((opt.User != currentUser && opt.User != "") || opt.Host != "" || (opt.Port != "22" && opt.Port != "")) { fs.Logf(name, "--sftp-ssh is in use - ignoring user/host/port from config - set in the parameters to --sftp-ssh (remove them from the config to silence this warning)") } + if opt.TranslateSymlinks && opt.SkipLinks { + return nil, errors.New("can't use --sftp-links and --sftp-skip-links together") + } f.tokens = pacer.NewTokenDispenser(opt.Connections) if opt.User == "" { @@ -1115,6 +1134,12 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m // Disable server side copy unless --sftp-copy-is-hardlink is set f.features.Copy = nil } + if opt.TranslateSymlinks { + // Enable symlink translation Feature when --sftp-links is set + // Not used yet but may be used in the future on shared backend operations. + // Maybe to check if src and dst backend support this feature before proceeding with the operation. + f.features.TranslateSymlink = true + } // Make a connection and pool it to return errors early c, err := f.getSftpConnection(ctx) if err != nil { @@ -1331,6 +1356,18 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e remote: remote, } o.setMetadata(info) + if o.IsSymlink() { + // Read the link target if it is a symlink to get path to real object and its size + linkTarget, linkTargetIsDir, sizeTarget, err := f.Readlink(ctx, o.path()) + if err != nil { + // If we can't read the link target, log the error and continue + fs.Errorf(remote, "Readlink failed: %v", err) + continue + } + o.size = sizeTarget + o.linkTarget = linkTarget + o.linkTargetIsDir = linkTargetIsDir + } entries = append(entries, o) } } @@ -1340,14 +1377,36 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Put data from into a new remote sftp file object described by and func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { err := f.mkParentDir(ctx, src.Remote()) + + // Init two variables to store symlink object and check if source object is a symlink + // If TranslateSymlinks is set + var srcSymlinkObject fs.Object + isSymlink := false + if err != nil { return nil, fmt.Errorf("Put mkParentDir failed: %w", err) } + + if f.opt.TranslateSymlinks { + // If TranslateSymlinks is set, we need to check if source object is a symlink + if or, ok := src.(*fs.OverrideRemote); ok { + srcSymlinkObject = or.UnWrap() + isSymlink = srcSymlinkObject.(*Object).IsSymlink() + } + } // Temporary object under construction o := &Object{ fs: f, remote: src.Remote(), } + // if source file is a symlink, we need to specify target path to temporary object + if isSymlink { + o.linkTarget = srcSymlinkObject.(*Object).linkTarget + o.size = srcSymlinkObject.(*Object).size + o.mode = srcSymlinkObject.(*Object).mode + o.linkTargetIsDir = srcSymlinkObject.(*Object).linkTargetIsDir + } + err = o.Update(ctx, in, src, options...) if err != nil { return nil, err @@ -1822,7 +1881,9 @@ func (o *Object) Remote() string { // Hash returns the selected checksum of the file // If no checksum is available it returns "" func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) { - if o.fs.opt.DisableHashCheck { + // Check if HashCheck is disabled or + // if TranslateSymlinks is enabled and the target is a directory + if o.fs.opt.DisableHashCheck || (o.fs.opt.TranslateSymlinks && o.linkTargetIsDir) { return "", nil } _ = o.fs.Hashes() @@ -1987,6 +2048,11 @@ func (o *Object) setMetadata(info os.FileInfo) { o.mode = info.Mode() } +// IsSymlink returns true if the remote sftp file is a symlink +func (o *Object) IsSymlink() bool { + return o.mode&os.ModeSymlink != 0 +} + // statRemote stats the file or directory at the remote given func (f *Fs) stat(ctx context.Context, remote string) (info os.FileInfo, err error) { absPath := remote @@ -1997,7 +2063,13 @@ func (f *Fs) stat(ctx context.Context, remote string) (info os.FileInfo, err err if err != nil { return nil, fmt.Errorf("stat: %w", err) } - info, err = c.sftpClient.Stat(absPath) + if f.opt.TranslateSymlinks { + // Lstat is used to get the info of the symlink itself instead of the target + // We use Lstat only if the user has requested --sftp-links flag + info, err = c.sftpClient.Lstat(absPath) + } else { + info, err = c.sftpClient.Stat(absPath) + } f.putSftpConnection(&c, err) return info, err } @@ -2041,8 +2113,13 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { return nil } -// Storable returns whether the remote sftp file is a regular file (not a directory, symbolic link, block device, character device, named pipe, etc.) +// Storable returns whether the remote sftp file is a regular file (not a directory, block device, character device, named pipe, etc.) +// if TranslateSymlinks is set, symlinks are also allowed func (o *Object) Storable() bool { + if o.fs.opt.TranslateSymlinks { + // If TranslateSymlinks is set, we also allow symlinks + return o.mode.IsRegular() || o.IsSymlink() + } return o.mode.IsRegular() } @@ -2156,12 +2233,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op if err != nil { return fmt.Errorf("Update: %w", err) } - // Hang on to the connection for the whole upload so it doesn't get reused while we are uploading - file, err := c.sftpClient.OpenFile(o.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC) - if err != nil { - o.fs.putSftpConnection(&c, err) - return fmt.Errorf("Update Create failed: %w", err) - } // remove the file if upload failed remove := func() { c, removeErr := o.fs.getSftpConnection(ctx) @@ -2177,6 +2248,44 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op fs.Debugf(src, "Removed after failed upload: %v", err) } } + // Check if TranslateSymlinks flag is set and if temporary object given by Put() is Symlink + if o.fs.opt.TranslateSymlinks && o.IsSymlink() { + // Create or update symlink + err = o.fs.Symlink(ctx, o.linkTarget, o.path()) + if err != nil { + o.fs.putSftpConnection(&c, err) + return fmt.Errorf("Update Symlink failed: %w", err) + } + _, linkTargetIsDir, sizeTarget, err := o.fs.Readlink(ctx, o.path()) + if o.linkTargetIsDir == linkTargetIsDir { + // if symlink target is a directory, in will be closed with ErrorReadIsDirectory + // FIXME : Needed to bypass closeErr := inAcc.Close() in updateOrPut function in copy.go <== Need Help here if bad practice + err = in.(*accounting.Account).Close() + if err == fs.ErrorReadIsDirectory { + fs.Debugf(o, "Readlink returned directory, Continue normally") + err = nil + } + } + if err != nil { + o.fs.putSftpConnection(&c, err) + remove() + return fmt.Errorf("Update Readlink failed: %w", err) + } + if sizeTarget != src.Size() { + o.fs.putSftpConnection(&c, err) + remove() + return fmt.Errorf("Update Readlink target's size mismatch: %d != %d", sizeTarget, src.Size()) + } + o.fs.putSftpConnection(&c, err) + o.size = sizeTarget + return nil + } + // Hang on to the connection for the whole upload so it doesn't get reused while we are uploading + file, err := c.sftpClient.OpenFile(o.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC) + if err != nil { + o.fs.putSftpConnection(&c, err) + return fmt.Errorf("Update Create failed: %w", err) + } _, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()}) if err != nil { o.fs.putSftpConnection(&c, err) @@ -2227,6 +2336,33 @@ func (o *Object) Remove(ctx context.Context) error { return err } +// Symlink creates a symbolic link on remote. +func (f *Fs) Symlink(ctx context.Context, targetPath, linkName string) error { + c, err := f.getSftpConnection(ctx) + if err != nil { + return fmt.Errorf("Symlink: %w", err) + } + err = c.sftpClient.Symlink(targetPath, linkName) + f.putSftpConnection(&c, err) + return err +} + +// Readlink reads the target of a symbolic link. +func (f *Fs) Readlink(ctx context.Context, path string) (string, bool, int64, error) { + c, err := f.getSftpConnection(ctx) + if err != nil { + return "", false, 0, err + } + target, err := c.sftpClient.Stat(path) + if err != nil { + f.putSftpConnection(&c, err) + return "", false, 0, err + } + link, err := c.sftpClient.ReadLink(path) + f.putSftpConnection(&c, err) + return link, target.IsDir(), target.Size(), err +} + // Check the interfaces are satisfied var ( _ fs.Fs = &Fs{} diff --git a/docs/content/sftp.md b/docs/content/sftp.md index 88d2eb76e..5b0b44c65 100644 --- a/docs/content/sftp.md +++ b/docs/content/sftp.md @@ -21,9 +21,9 @@ SSH installations. Paths are specified as `remote:path`. If the path does not begin with a `/` it is relative to the home directory of the user. An empty path -`remote:` refers to the user's home directory. For example, `rclone lsd remote:` -would list the home directory of the user configured in the rclone remote config -(`i.e /home/sftpuser`). However, `rclone lsd remote:/` would list the root +`remote:` refers to the user's home directory. For example, `rclone lsd remote:` +would list the home directory of the user configured in the rclone remote config +(`i.e /home/sftpuser`). However, `rclone lsd remote:/` would list the root directory for remote machine (i.e. `/`) Note that some SFTP servers will need the leading / - Synology is a @@ -128,7 +128,7 @@ The SFTP remote supports three authentication methods: Key files should be PEM-encoded private key files. For instance `/home/$USER/.ssh/id_rsa`. Only unencrypted OpenSSH or PEM encrypted files are supported. -The key file can be specified in either an external file (key_file) or contained within the +The key file can be specified in either an external file (key_file) or contained within the rclone config file (key_pem). If using key_pem in the config file, the entry should be on a single line with new line ('\n' or '\r\n') separating lines. i.e. @@ -199,7 +199,7 @@ e.g. using the OpenSSH `known_hosts` file: type = sftp host = example.com user = sftpuser -pass = +pass = known_hosts_file = ~/.ssh/known_hosts ```` @@ -593,7 +593,7 @@ Properties: - Config: ssh - Env Var: RCLONE_SFTP_SSH - Type: SpaceSepList -- Default: +- Default: ### Advanced options @@ -647,13 +647,13 @@ E.g. if shared folders can be found in directories representing volumes: E.g. if home directory can be found in a shared folder called "home": rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory - + To specify only the path to the SFTP remote's root, and allow rclone to add any relative subpaths automatically (including unwrapping/decrypting remotes as necessary), add the '@' character to the beginning of the path. E.g. the first example above could be rewritten as: rclone sync /home/local/directory remote:/directory --sftp-path-override @/volume2 - + Note that when using this method with Synology "home" folders, the full "/homes/USER" path should be specified instead of "/home". E.g. the second example above should be rewritten as: @@ -730,6 +730,8 @@ Properties: Set to skip any symlinks and any other non regular files. +Mutually exclusive with `--sftp-links`. + Properties: - Config: skip_links @@ -737,6 +739,19 @@ Properties: - Type: bool - Default: false +#### --sftp-links + +Set to follow symlinks. Recreate or update symlinks as symlinks and not copy the underlying file/directory. + +Mutually exclusive with `--sftp-skip-links`. + +Properties: + +- Config: links +- Env Var: RCLONE_SFTP_LINKS +- Type: bool +- Default: false + #### --sftp-subsystem Specifies the SSH2 subsystem on the remote host. @@ -754,7 +769,7 @@ Specifies the path or command to run a sftp server on the remote host. The subsystem option is ignored when server_command is defined. -If adding server_command to the configuration file please note that +If adding server_command to the configuration file please note that it should not be enclosed in quotes, since that will make rclone fail. A working example is: @@ -946,7 +961,7 @@ Properties: - Config: set_env - Env Var: RCLONE_SFTP_SET_ENV - Type: SpaceSepList -- Default: +- Default: #### --sftp-ciphers @@ -966,7 +981,7 @@ Properties: - Config: ciphers - Env Var: RCLONE_SFTP_CIPHERS - Type: SpaceSepList -- Default: +- Default: #### --sftp-key-exchange @@ -986,7 +1001,7 @@ Properties: - Config: key_exchange - Env Var: RCLONE_SFTP_KEY_EXCHANGE - Type: SpaceSepList -- Default: +- Default: #### --sftp-macs @@ -1004,7 +1019,7 @@ Properties: - Config: macs - Env Var: RCLONE_SFTP_MACS - Type: SpaceSepList -- Default: +- Default: #### --sftp-host-key-algorithms @@ -1024,18 +1039,18 @@ Properties: - Config: host_key_algorithms - Env Var: RCLONE_SFTP_HOST_KEY_ALGORITHMS - Type: SpaceSepList -- Default: +- Default: #### --sftp-socks-proxy Socks 5 proxy host. - + Supports the format user:pass@host:port, user@host:port, host:port. Example: myUser:myPass@localhost:9005 - + Properties: diff --git a/fs/features.go b/fs/features.go index 828f3b94b..a28d77001 100644 --- a/fs/features.go +++ b/fs/features.go @@ -39,6 +39,7 @@ type Features struct { NoMultiThreading bool // set if can't have multiplethreads on one download open Overlay bool // this wraps one or more backends to add functionality ChunkWriterDoesntSeek bool // set if the chunk writer doesn't need to read the data more than once + TranslateSymlink bool // set if backend can Read and Write symlinks // Purge all files in the directory specified // diff --git a/fs/fs.go b/fs/fs.go index fe68517c4..53cc4628f 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -48,6 +48,7 @@ var ( ErrorNotImplemented = errors.New("optional feature not implemented") ErrorCommandNotFound = errors.New("command not found") ErrorFileNameTooLong = errors.New("file name too long") + ErrorReadIsDirectory = errors.New("read is a directory") ) // CheckClose is a utility function used to check the return from