extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-12-22 07:01:46 +01:00
Code Issues Packages Projects Releases Wiki Activity

vfs: add symlink support to VFS

Browse Source 
This is somewhat limited in that it only resolves symlinks when files
are opened. This will work fine for the intended use in rclone mount,
but is inadequate for the other servers probably.
This commit is contained in:
Nick Craig-Wood 2024-10-17 16:18:37 +01:00
parent c0339327be
commit a5abe4b8b3
5 changed files with 268 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
vfs/dir.go
View File

@ -459,7 +459,8 @@ func (d *Dir) addObject(node Node) {
// This will be replaced with a real object when it is read back from the
// remote.
//
// This is used to add directory entries while things are uploading
// This is used by the vfs cache to insert objects that are uploading
// into the directory tree.
func (d *Dir) AddVirtual(leaf string, size int64, isDir bool) {
var node Node
d.mu.RLock()
@ -475,7 +476,16 @@ func (d *Dir) AddVirtual(leaf string, size int64, isDir bool) {
entry := fs.NewDir(remote, time.Now())
node = newDir(d.vfs, d.f, d, entry)
} else {
isLink := false
if d.vfs.Opt.Links {
// since the path came from the cache it may have fs.LinkSuffix,
// so remove it and mark the *File accordingly
leaf, isLink = strings.CutSuffix(leaf, fs.LinkSuffix)
}
f := newFile(d, dPath, nil, leaf)
if isLink {
f.setSymlink()
}
f.setSize(size)
node = f
}
@ -628,7 +638,7 @@ func (d *Dir) _purgeVirtual() {
// if writing in progress then leave virtual
continue
}
if d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal && d.vfs.cache.InUse(f.Path()) {
if d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal && d.vfs.cache.InUse(f.CachePath()) {
// if object in use or dirty then leave virtual
continue
}
@ -718,6 +728,9 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree
if name == "." || name == ".." {
continue
}
if d.vfs.Opt.Links {
name, _ = strings.CutSuffix(name, fs.LinkSuffix)
}
node := d.items[name]
if mv.add(d, name) {
continue

177
vfs/file.go
View File

@ -4,8 +4,10 @@ import (
"context"
"errors"
"fmt"
"io"
"os"
"path"
"strings"
"sync"
"sync/atomic"
"time"
@ -34,7 +36,7 @@ import (
//
// File may **not** read any members of Dir directly.
// File represents a file
// File represents a file or a symlink
type File struct {
inode uint64 // inode number - read only
size atomic.Int64 // size of file
@ -53,6 +55,7 @@ type File struct {
sys atomic.Value // user defined info to be attached here
nwriters atomic.Int32 // len(writers)
appendMode bool // file was opened with O_APPEND
isLink bool // file represents a symlink
}
// newFile creates a new File
@ -69,9 +72,18 @@ func newFile(d *Dir, dPath string, o fs.Object, leaf string) *File {
if o != nil {
f.size.Store(o.Size())
}
f._setIsLink()
return f
}
// Set whether this is a link or not based on f.o
func (f *File) _setIsLink() {
if f.o == nil {
return
}
f.isLink = f.d.vfs.Opt.Links && strings.HasSuffix(f.o.Remote(), fs.LinkSuffix)
}
// String converts it to printable
func (f *File) String() string {
if f == nil {
@ -90,13 +102,31 @@ func (f *File) IsDir() bool {
return false
}
// IsSymlink returns true for symlinks when --links is enabled
func (f *File) IsSymlink() bool {
f.mu.RLock()
defer f.mu.RUnlock()
return f.isLink
}
// setSymlink marks this File as being a symlink
func (f *File) setSymlink() {
f.mu.RLock()
f.isLink = true
f.mu.RUnlock()
}
// Mode bits of the file or directory - satisfies Node interface
func (f *File) Mode() (mode os.FileMode) {
f.mu.RLock()
defer f.mu.RUnlock()
mode = os.FileMode(f.d.vfs.Opt.FilePerms)
if f.appendMode {
mode |= os.ModeAppend
if f.isLink {
mode = os.FileMode(f.d.vfs.Opt.LinkPerms)
} else {
mode = os.FileMode(f.d.vfs.Opt.FilePerms)
if f.appendMode {
mode |= os.ModeAppend
}
}
return mode
}
@ -122,6 +152,34 @@ func (f *File) Path() string {
return path.Join(dPath, leaf)
}
// _fixCachePath returns fullPath with the fs.LinkSuffix added if appropriate
// use when lock is held
func (f *File) _fixCachePath(fullPath string) string {
if !f.isLink {
return fullPath
}
return fullPath + fs.LinkSuffix
}
// _cachePath returns the full path of the file with the fs.LinkSuffix if appropriate
// use when lock is held
func (f *File) _cachePath() string {
dPath, leaf := f.dPath, f.leaf
if f.isLink {
leaf += fs.LinkSuffix
}
return path.Join(dPath, leaf)
}
// CachePath returns the full path of the file with the fs.LinkSuffix if appropriate
//
// We use this path when storing files in the cache.
func (f *File) CachePath() string {
f.mu.RLock()
defer f.mu.RUnlock()
return f._cachePath()
}
// Sys returns underlying data source (can be nil) - satisfies Node interface
func (f *File) Sys() interface{} {
return f.sys.Load()
@ -172,6 +230,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
f.mu.RLock()
d := f.d
oldPendingRenameFun := f.pendingRenameFun
oldPath := f._cachePath()
newCacheName := f._fixCachePath(newName)
f.mu.RUnlock()
if features := d.Fs().Features(); features.Move == nil && features.Copy == nil {
@ -180,9 +240,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
return err
}
oldPath := f.Path()
// File.mu is unlocked here to call Dir.Path()
newPath := path.Join(destDir.Path(), newName)
newPath := path.Join(destDir.Path(), newCacheName)
renameCall := func(ctx context.Context) (err error) {
// chain rename calls if any
@ -231,6 +290,7 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
f.mu.Lock()
if newObject != nil {
f.o = newObject
f._setIsLink()
}
f.pendingRenameFun = nil
f.mu.Unlock()
@ -334,7 +394,7 @@ func (f *File) ModTime() (modTime time.Time) {
}
// Read the modtime from a dirty item if it exists
if f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal {
if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil {
if item := f.d.vfs.cache.DirtyItem(f._cachePath()); item != nil {
modTime, err := item.GetModTime()
if err != nil {
fs.Errorf(f._path(), "ModTime: Item GetModTime failed: %v", err)
@ -371,7 +431,7 @@ func (f *File) Size() int64 {
// Read the size from a dirty item if it exists
if f.d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal {
if item := f.d.vfs.cache.DirtyItem(f._path()); item != nil {
if item := f.d.vfs.cache.DirtyItem(f._cachePath()); item != nil {
size, err := item.GetSize()
if err != nil {
fs.Errorf(f._path(), "Size: Item GetSize failed: %v", err)
@ -404,8 +464,8 @@ func (f *File) SetModTime(modTime time.Time) error {
f.pendingModTime = modTime
// set the time of the file in the cache
if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f._path()) {
f.d.vfs.cache.SetModTime(f._path(), f.pendingModTime)
if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f._cachePath()) {
f.d.vfs.cache.SetModTime(f._cachePath(), f.pendingModTime)
}
// Only update the ModTime when there are no writers, setObject will do it
@ -480,6 +540,7 @@ func (f *File) setSize(n int64) {
func (f *File) setObject(o fs.Object) {
f.mu.Lock()
f.o = o
f._setIsLink()
_ = f._applyPendingModTime()
d := f.d
f.mu.Unlock()
@ -493,6 +554,7 @@ func (f *File) setObject(o fs.Object) {
func (f *File) setObjectNoUpdate(o fs.Object) {
f.mu.Lock()
f.o = o
f._setIsLink()
f.virtualModTime = nil
fs.Debugf(f._path(), "Reset virtual modtime")
f.mu.Unlock()
@ -611,8 +673,8 @@ func (f *File) Remove() (err error) {
// Remove the object from the cache
wasWriting := false
if d.vfs.cache != nil && d.vfs.cache.Exists(f.Path()) {
wasWriting = d.vfs.cache.Remove(f.Path())
if d.vfs.cache != nil && d.vfs.cache.Exists(f.CachePath()) {
wasWriting = d.vfs.cache.Remove(f.CachePath())
}
f.muRW.Lock() // muRW must be locked before mu to avoid
@ -673,6 +735,85 @@ func (f *File) Fs() fs.Fs {
return f.d.Fs()
}
// MaxSymlinkIterations is the largest number of symlink evaluations EvalSymlinks will do.
const MaxSymlinkIterations = 32
// If f is a symlink then it resolves it to a new Node.
//
// This is a simplistic symlink resolver - it only resolves direct
// symlinks, it will **not** resolve paths that point into a directory
// via a symlink.
//
// It returns the target node after the evaluation of all symbolic
// links.
//
// It returns an error if too many symlinks need to be resolved
// (ELOOP) or there is a loop.
func (f *File) resolveNode() (target Node, err error) {
defer log.Trace(f.Path(), "")("target=%v, err=%v", &target, &err)
seen := make(map[string]struct{})
for tries := 0; tries < MaxSymlinkIterations; tries++ {
// If f isn't a symlink, we've arrived at the target
if !f.IsSymlink() {
return f, nil
}
// Read the symlink
fd, err := f.Open(os.O_RDONLY | o_SYMLINK)
if err != nil {
return nil, err
}
b, err := io.ReadAll(fd)
closeErr := fd.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
targetPath := string(b)
// Convert to a path relative to the root
// Symlinks are relative to their file node
if !path.IsAbs(targetPath) {
basePath := path.Dir(f.Path())
targetPath = path.Join(basePath, targetPath)
}
// Clean the path, rclone style
targetPath = path.Clean(targetPath)
if targetPath == "." {
targetPath = ""
}
// Check if we've already seen this path
if _, ok := seen[targetPath]; ok {
return nil, ELOOP
}
seen[targetPath] = struct{}{}
// Resolve the targetPath into a node
target, err = f.d.vfs.Stat(targetPath)
if err != nil {
return nil, err
}
// Return node as it must be the destination if not a file
var ok bool
f, ok = target.(*File)
if !ok {
return target, nil
}
}
return nil, ELOOP
}
// Open also also implements the internal flag o_SYMLINK which instead
// of opening the file a symlink points to, opens the symlink itself.
// This is used for reading and writing the symlink and shouldn't be
// used externally.
const o_SYMLINK = 0x4000_0000 //nolint:revive
// Open a file according to the flags provided
//
// O_RDONLY open the file read-only.
@ -694,6 +835,16 @@ func (f *File) Open(flags int) (fd Handle, err error) {
rdwrMode = flags & accessModeMask
)
// If this is a symlink, then resolve it
if f.IsSymlink() && flags&o_SYMLINK == 0 {
target, err := f.resolveNode()
if err != nil {
return nil, err
}
return target.Open(flags)
}
flags &^= o_SYMLINK
// http://pubs.opengroup.org/onlinepubs/7908799/xsh/open.html
// The result of using O_TRUNC with O_RDONLY is undefined.
// Linux seems to truncate the file, but we prefer to return EINVAL
@ -738,7 +889,7 @@ func (f *File) Open(flags int) (fd Handle, err error) {
d := f.d
f.mu.RUnlock()
CacheMode := d.vfs.Opt.CacheMode
if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.InUse(f.Path()) || d.vfs.cache.Exists(f.Path())) {
if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.InUse(f.CachePath()) || d.vfs.cache.Exists(f.CachePath())) {
fd, err = f.openRW(flags)
} else if read && write {
if CacheMode >= vfscommon.CacheModeMinimal {

2
vfs/read_write.go
View File

@ -43,7 +43,7 @@ func (fh *RWFileHandle) Unlock() error {
func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) {
defer log.Trace(f.Path(), "")("err=%v", &err)
// get an item to represent this from the cache
item := d.vfs.cache.Item(f.Path())
item := d.vfs.cache.Item(f.CachePath())
exists := f.exists() || (item.Exists() && !item.WrittenBack())

85
vfs/vfs.go
View File

@ -791,6 +791,9 @@ func (vfs *VFS) WriteFile(name string, data []byte, perm os.FileMode) (err error
}
// AddVirtual adds the object (file or dir) to the directory cache
//
// This is used by the vfs cache to insert objects that are uploading
// into the directory tree.
func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) (err error) {
remote = strings.TrimRight(remote, "/")
var dir *Dir
@ -808,3 +811,85 @@ func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) (err error) {
dir.AddVirtual(leaf, size, false)
return nil
}
// Readlink returns the destination of the named symbolic link.
// If there is an error, it will be of type *PathError.
func (vfs *VFS) Readlink(name string) (s string, err error) {
if !vfs.Opt.Links {
fs.Errorf(nil, "symlinks not supported without the --links flag: %v", name)
return "", ENOSYS
}
node, err := vfs.Stat(name)
if err != nil {
return "", err
}
file, ok := node.(*File)
if !ok || !file.IsSymlink() {
return "", EINVAL // not a symlink
}
fd, err := file.Open(os.O_RDONLY | o_SYMLINK)
if err != nil {
return "", err
}
defer fs.CheckClose(fd, &err)
b, err := io.ReadAll(fd)
if err != nil {
return "", err
}
return string(b), nil
}
// CreateSymlink creates newname as a symbolic link to oldname.
// On Windows, a symlink to a non-existent oldname creates a file symlink;
// if oldname is later created as a directory the symlink will not work.
// It returns the node created
func (vfs *VFS) CreateSymlink(oldname, newname string) (Node, error) {
if !vfs.Opt.Links {
fs.Errorf(newname, "symlinks not supported without the --links flag")
return nil, ENOSYS
}
// Destination can't exist
_, err := vfs.Stat(newname)
if err == nil {
return nil, EEXIST
} else if err != ENOENT {
return nil, err
}
// Find the parent
dir, leaf, err := vfs.StatParent(newname)
if err != nil {
return nil, err
}
// Create the file node
flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | o_SYMLINK
file, err := dir.Create(leaf, flags)
if err != nil {
return nil, err
}
// Force the file to be a link
file.setSymlink()
// Open the file
fh, err := file.Open(flags)
if err != nil {
return nil, err
}
defer fs.CheckClose(fh, &err)
// Write the symlink data
_, err = fh.Write([]byte(strings.ReplaceAll(oldname, "\\", "/")))
return file, nil
}
// Symlink creates newname as a symbolic link to oldname.
// On Windows, a symlink to a non-existent oldname creates a file symlink;
// if oldname is later created as a directory the symlink will not work.
func (vfs *VFS) Symlink(oldname, newname string) error {
_, err := vfs.CreateSymlink(oldname, newname)
return err
}

3
vfs/write.go
View File

@ -37,6 +37,9 @@ var (
)
func newWriteFileHandle(d *Dir, f *File, remote string, flags int) (*WriteFileHandle, error) {
if f.IsSymlink() {
remote += fs.LinkSuffix
}
fh := &WriteFileHandle{
remote: remote,
flags: flags,