From c9374fbe5a0f5b4eb843e7865f6400472a7858b3 Mon Sep 17 00:00:00 2001 From: Max Sum Date: Fri, 31 Jan 2020 11:34:46 +0800 Subject: [PATCH] union: fix issues when using space-relavant and path-preserving policies Path-preserving policy would need to look for the parent dir of operating path. Therefor if the operating path is the same path as root passed in during NewFs, there would be no room for uplooking. And also About() might have problem if the folder is no exist. RootFs is added to solve this problem. --- backend/union/policy/epall.go | 7 +++- backend/union/policy/epff.go | 7 +++- backend/union/policy/newest.go | 7 +++- backend/union/union.go | 61 +++++++++++++++++++----------- backend/union/upstream/upstream.go | 30 +++++++++------ 5 files changed, 72 insertions(+), 40 deletions(-) diff --git a/backend/union/policy/epall.go b/backend/union/policy/epall.go index 63b994b3f..3f765c155 100644 --- a/backend/union/policy/epall.go +++ b/backend/union/policy/epall.go @@ -2,6 +2,7 @@ package policy import ( "context" + "path/filepath" "sync" "github.com/rclone/rclone/backend/union/upstream" @@ -27,7 +28,9 @@ func (p *EpAll) epall(ctx context.Context, upstreams []*upstream.Fs, path string wg.Add(1) i, u := i, u // Closure go func() { - if findEntry(ctx, u, path) != nil { + rfs := u.RootFs + remote := filepath.Join(u.RootPath, path) + if findEntry(ctx, rfs, remote) != nil { ufs[i] = u } wg.Done() @@ -79,7 +82,7 @@ func (p *EpAll) Create(ctx context.Context, upstreams []*upstream.Fs, path strin if len(upstreams) == 0 { return nil, fs.ErrorPermissionDenied } - upstreams, err := p.epall(ctx, upstreams, path) + upstreams, err := p.epall(ctx, upstreams, path+"/..") return upstreams, err } diff --git a/backend/union/policy/epff.go b/backend/union/policy/epff.go index b93afbc1b..dd7816ba7 100644 --- a/backend/union/policy/epff.go +++ b/backend/union/policy/epff.go @@ -2,6 +2,7 @@ package policy import ( "context" + "path/filepath" "github.com/rclone/rclone/backend/union/upstream" "github.com/rclone/rclone/fs" @@ -20,7 +21,9 @@ func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, path string) for _, u := range upstreams { u := u // Closure go func() { - if findEntry(ctx, u, path) == nil { + rfs := u.RootFs + remote := filepath.Join(u.RootPath, path) + if findEntry(ctx, rfs, remote) == nil { u = nil } ch <- u @@ -79,7 +82,7 @@ func (p *EpFF) Create(ctx context.Context, upstreams []*upstream.Fs, path string if len(upstreams) == 0 { return nil, fs.ErrorPermissionDenied } - u, err := p.epff(ctx, upstreams, path) + u, err := p.epff(ctx, upstreams, path+"/..") return []*upstream.Fs{u}, err } diff --git a/backend/union/policy/newest.go b/backend/union/policy/newest.go index c8fd4f47a..1276955ad 100644 --- a/backend/union/policy/newest.go +++ b/backend/union/policy/newest.go @@ -2,6 +2,7 @@ package policy import ( "context" + "path/filepath" "sync" "time" @@ -28,7 +29,9 @@ func (p *Newest) newest(ctx context.Context, upstreams []*upstream.Fs, path stri i, u := i, u // Closure go func() { defer wg.Done() - if e := findEntry(ctx, u, path); e != nil { + rfs := u.RootFs + remote := filepath.Join(u.RootPath, path) + if e := findEntry(ctx, rfs, remote); e != nil { ufs[i] = u mtimes[i] = e.ModTime(ctx) } @@ -112,7 +115,7 @@ func (p *Newest) Create(ctx context.Context, upstreams []*upstream.Fs, path stri if len(upstreams) == 0 { return nil, fs.ErrorPermissionDenied } - u, err := p.newest(ctx, upstreams, path) + u, err := p.newest(ctx, upstreams, path+"/..") return []*upstream.Fs{u}, err } diff --git a/backend/union/union.go b/backend/union/union.go index 2a1a21930..ea00cb842 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -15,7 +15,6 @@ import ( "github.com/rclone/rclone/backend/union/policy" "github.com/rclone/rclone/backend/union/upstream" "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/hash" @@ -143,21 +142,12 @@ func (f *Fs) Hashes() hash.Set { // Mkdir makes the root directory of the Fs object func (f *Fs) Mkdir(ctx context.Context, dir string) error { - var upstreams []*upstream.Fs - var err error - if dir == "" { - pf, e := cache.Get(f.name + ":") - if e != nil { - return e + upstreams, err := f.create(ctx, dir) + if err == fs.ErrorObjectNotFound && dir != parentDir(dir) { + if err := f.Mkdir(ctx, parentDir(dir)); err != nil { + return err } - pfs, ok := pf.(*Fs) - if !ok { - return errors.New("failed to get parent Fs") - } - upstreams, err = pfs.create(ctx, "") - dir = f.root - } else { - upstreams, err = f.create(ctx, parentDir(dir)) + upstreams, err = f.create(ctx, dir) } if err != nil { return err @@ -259,7 +249,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, errs[i] = errors.Wrap(fs.ErrorNotAFile, u.Name()) return } - mo, err := u.Features().Move(ctx, o, remote) + mo, err := u.Features().Move(ctx, o.UnWrap(), remote) if err != nil || mo == nil { errs[i] = errors.Wrap(err, u.Name()) return @@ -288,12 +278,12 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, // // If destination exists then return fs.ErrorDirExists func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { - _, ok := src.(*Fs) + sfs, ok := src.(*Fs) if !ok { fs.Debugf(src, "Can't move directory - not same remote type") return fs.ErrorCantDirMove } - upstreams, err := f.action(ctx, srcRemote) + upstreams, err := sfs.action(ctx, srcRemote) if err != nil { return err } @@ -304,11 +294,30 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string } errs := Errors(make([]error, len(upstreams))) multithread(len(upstreams), func(i int) { - u := upstreams[i] - err := u.Features().DirMove(ctx, u, srcRemote, dstRemote) - errs[i] = errors.Wrap(err, u.Name()) + su := upstreams[i] + var du *upstream.Fs + for _, u := range f.upstreams { + if u.RootFs.Root() == su.RootFs.Root() { + du = u + } + } + if du == nil { + errs[i] = errors.Wrap(fs.ErrorCantDirMove, su.Name()+":"+su.Root()) + return + } + err := du.Features().DirMove(ctx, su.Fs, srcRemote, dstRemote) + errs[i] = errors.Wrap(err, du.Name()+":"+du.Root()) }) - return errs.Err() + errs = errs.FilterNil() + if len(errs) == 0 { + return nil + } + for _, e := range errs { + if errors.Cause(e) != fs.ErrorDirExists { + return errs + } + } + return fs.ErrorDirExists } // ChangeNotify calls the passed function with a path @@ -355,7 +364,13 @@ func (f *Fs) DirCacheFlush() { func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) { srcPath := src.Remote() - upstreams, err := f.create(ctx, parentDir(srcPath)) + upstreams, err := f.create(ctx, srcPath) + if err == fs.ErrorObjectNotFound { + if err := f.Mkdir(ctx, parentDir(srcPath)); err != nil { + return nil, err + } + upstreams, err = f.create(ctx, srcPath) + } if err != nil { return nil, err } diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go index 9737b7249..7543cfad0 100644 --- a/backend/union/upstream/upstream.go +++ b/backend/union/upstream/upstream.go @@ -23,6 +23,8 @@ var ( // Fs is a wrap of any fs and its configs type Fs struct { fs.Fs + RootFs fs.Fs + RootPath string writable bool creatable bool usage *fs.Usage // Cache the usage @@ -63,7 +65,8 @@ func New(remote, root string, cacheTime time.Duration) (*Fs, error) { if err != nil { return nil, err } - rFs := &Fs{ + f := &Fs{ + RootPath: root, writable: true, creatable: true, cacheExpiry: time.Now().Unix(), @@ -71,24 +74,29 @@ func New(remote, root string, cacheTime time.Duration) (*Fs, error) { usage: &fs.Usage{}, } if strings.HasSuffix(fsPath, ":ro") { - rFs.writable = false - rFs.creatable = false + f.writable = false + f.creatable = false fsPath = fsPath[0 : len(fsPath)-3] } else if strings.HasSuffix(fsPath, ":nc") { - rFs.writable = true - rFs.creatable = false + f.writable = true + f.creatable = false fsPath = fsPath[0 : len(fsPath)-3] } - var rootString = path.Join(fsPath, filepath.ToSlash(root)) if configName != "local" { - rootString = configName + ":" + rootString + fsPath = configName + ":" + fsPath } + rFs, err := cache.Get(fsPath) + if err != nil && err != fs.ErrorIsFile { + return nil, err + } + f.RootFs = rFs + rootString := path.Join(fsPath, filepath.ToSlash(root)) myFs, err := cache.Get(rootString) if err != nil && err != fs.ErrorIsFile { return nil, err } - rFs.Fs = myFs - return rFs, err + f.Fs = myFs + return f, err } // WrapDirectory wraps a fs.Directory to include the info @@ -274,7 +282,7 @@ func (f *Fs) GetUsedSpace() (int64, error) { } func (f *Fs) updateUsage() (err error) { - if do := f.Fs.Features().About; do == nil { + if do := f.RootFs.Features().About; do == nil { return ErrUsageFieldNotSupported } done := false @@ -301,7 +309,7 @@ func (f *Fs) updateUsageCore(lock bool) error { // Run in background, should not be cancelled by user ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() - usage, err := f.Features().About(ctx) + usage, err := f.RootFs.Features().About(ctx) if err != nil { f.cacheUpdate = false return err