extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-01-31 10:41:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

union: fix issues when using space-relavant and path-preserving policies

Browse Source 
Path-preserving policy would need to look for the parent dir of operating path. Therefor if the operating path is the
same path as root passed in during NewFs, there would be no room for uplooking. And also About() might have
problem if the folder is no exist. RootFs is added to solve this problem.
This commit is contained in:
Max Sum 2020-01-31 11:34:46 +08:00 committed by Nick Craig-Wood
parent 0081971ade
commit c9374fbe5a
5 changed files with 72 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
backend/union/policy/epall.go
View File

@ -2,6 +2,7 @@ package policy
import (
"context"
"path/filepath"
"sync"
"github.com/rclone/rclone/backend/union/upstream"
@ -27,7 +28,9 @@ func (p *EpAll) epall(ctx context.Context, upstreams []*upstream.Fs, path string
wg.Add(1)
i, u := i, u // Closure
go func() {
if findEntry(ctx, u, path) != nil {
rfs := u.RootFs
remote := filepath.Join(u.RootPath, path)
if findEntry(ctx, rfs, remote) != nil {
ufs[i] = u
}
wg.Done()
@ -79,7 +82,7 @@ func (p *EpAll) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
if len(upstreams) == 0 {
return nil, fs.ErrorPermissionDenied
}
upstreams, err := p.epall(ctx, upstreams, path)
upstreams, err := p.epall(ctx, upstreams, path+"/..")
return upstreams, err
}

7
backend/union/policy/epff.go
View File

@ -2,6 +2,7 @@ package policy
import (
"context"
"path/filepath"
"github.com/rclone/rclone/backend/union/upstream"
"github.com/rclone/rclone/fs"
@ -20,7 +21,9 @@ func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, path string)
for _, u := range upstreams {
u := u // Closure
go func() {
if findEntry(ctx, u, path) == nil {
rfs := u.RootFs
remote := filepath.Join(u.RootPath, path)
if findEntry(ctx, rfs, remote) == nil {
u = nil
}
ch <- u
@ -79,7 +82,7 @@ func (p *EpFF) Create(ctx context.Context, upstreams []*upstream.Fs, path string
if len(upstreams) == 0 {
return nil, fs.ErrorPermissionDenied
}
u, err := p.epff(ctx, upstreams, path)
u, err := p.epff(ctx, upstreams, path+"/..")
return []*upstream.Fs{u}, err
}

7
backend/union/policy/newest.go
View File

@ -2,6 +2,7 @@ package policy
import (
"context"
"path/filepath"
"sync"
"time"
@ -28,7 +29,9 @@ func (p *Newest) newest(ctx context.Context, upstreams []*upstream.Fs, path stri
i, u := i, u // Closure
go func() {
defer wg.Done()
if e := findEntry(ctx, u, path); e != nil {
rfs := u.RootFs
remote := filepath.Join(u.RootPath, path)
if e := findEntry(ctx, rfs, remote); e != nil {
ufs[i] = u
mtimes[i] = e.ModTime(ctx)
}
@ -112,7 +115,7 @@ func (p *Newest) Create(ctx context.Context, upstreams []*upstream.Fs, path stri
if len(upstreams) == 0 {
return nil, fs.ErrorPermissionDenied
}
u, err := p.newest(ctx, upstreams, path)
u, err := p.newest(ctx, upstreams, path+"/..")
return []*upstream.Fs{u}, err
}

61
backend/union/union.go
View File

@ -15,7 +15,6 @@ import (
"github.com/rclone/rclone/backend/union/policy"
"github.com/rclone/rclone/backend/union/upstream"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/hash"
@ -143,21 +142,12 @@ func (f *Fs) Hashes() hash.Set {
// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
var upstreams []*upstream.Fs
var err error
if dir == "" {
pf, e := cache.Get(f.name + ":")
if e != nil {
return e
upstreams, err := f.create(ctx, dir)
if err == fs.ErrorObjectNotFound && dir != parentDir(dir) {
if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
return err
}
pfs, ok := pf.(*Fs)
if !ok {
return errors.New("failed to get parent Fs")
}
upstreams, err = pfs.create(ctx, "")
dir = f.root
} else {
upstreams, err = f.create(ctx, parentDir(dir))
upstreams, err = f.create(ctx, dir)
}
if err != nil {
return err
@ -259,7 +249,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
errs[i] = errors.Wrap(fs.ErrorNotAFile, u.Name())
return
}
mo, err := u.Features().Move(ctx, o, remote)
mo, err := u.Features().Move(ctx, o.UnWrap(), remote)
if err != nil || mo == nil {
errs[i] = errors.Wrap(err, u.Name())
return
@ -288,12 +278,12 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
//
// If destination exists then return fs.ErrorDirExists
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
_, ok := src.(*Fs)
sfs, ok := src.(*Fs)
if !ok {
fs.Debugf(src, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove
}
upstreams, err := f.action(ctx, srcRemote)
upstreams, err := sfs.action(ctx, srcRemote)
if err != nil {
return err
}
@ -304,11 +294,30 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
}
errs := Errors(make([]error, len(upstreams)))
multithread(len(upstreams), func(i int) {
u := upstreams[i]
err := u.Features().DirMove(ctx, u, srcRemote, dstRemote)
errs[i] = errors.Wrap(err, u.Name())
su := upstreams[i]
var du *upstream.Fs
for _, u := range f.upstreams {
if u.RootFs.Root() == su.RootFs.Root() {
du = u
}
}
if du == nil {
errs[i] = errors.Wrap(fs.ErrorCantDirMove, su.Name()+":"+su.Root())
return
}
err := du.Features().DirMove(ctx, su.Fs, srcRemote, dstRemote)
errs[i] = errors.Wrap(err, du.Name()+":"+du.Root())
})
return errs.Err()
errs = errs.FilterNil()
if len(errs) == 0 {
return nil
}
for _, e := range errs {
if errors.Cause(e) != fs.ErrorDirExists {
return errs
}
}
return fs.ErrorDirExists
}
// ChangeNotify calls the passed function with a path
@ -355,7 +364,13 @@ func (f *Fs) DirCacheFlush() {
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) {
srcPath := src.Remote()
upstreams, err := f.create(ctx, parentDir(srcPath))
upstreams, err := f.create(ctx, srcPath)
if err == fs.ErrorObjectNotFound {
if err := f.Mkdir(ctx, parentDir(srcPath)); err != nil {
return nil, err
}
upstreams, err = f.create(ctx, srcPath)
}
if err != nil {
return nil, err
}

30
backend/union/upstream/upstream.go
View File

@ -23,6 +23,8 @@ var (
// Fs is a wrap of any fs and its configs
type Fs struct {
fs.Fs
RootFs fs.Fs
RootPath string
writable bool
creatable bool
usage *fs.Usage // Cache the usage
@ -63,7 +65,8 @@ func New(remote, root string, cacheTime time.Duration) (*Fs, error) {
if err != nil {
return nil, err
}
rFs := &Fs{
f := &Fs{
RootPath: root,
writable: true,
creatable: true,
cacheExpiry: time.Now().Unix(),
@ -71,24 +74,29 @@ func New(remote, root string, cacheTime time.Duration) (*Fs, error) {
usage: &fs.Usage{},
}
if strings.HasSuffix(fsPath, ":ro") {
rFs.writable = false
rFs.creatable = false
f.writable = false
f.creatable = false
fsPath = fsPath[0 : len(fsPath)-3]
} else if strings.HasSuffix(fsPath, ":nc") {
rFs.writable = true
rFs.creatable = false
f.writable = true
f.creatable = false
fsPath = fsPath[0 : len(fsPath)-3]
}
var rootString = path.Join(fsPath, filepath.ToSlash(root))
if configName != "local" {
rootString = configName + ":" + rootString
fsPath = configName + ":" + fsPath
}
rFs, err := cache.Get(fsPath)
if err != nil && err != fs.ErrorIsFile {
return nil, err
}
f.RootFs = rFs
rootString := path.Join(fsPath, filepath.ToSlash(root))
myFs, err := cache.Get(rootString)
if err != nil && err != fs.ErrorIsFile {
return nil, err
}
rFs.Fs = myFs
return rFs, err
f.Fs = myFs
return f, err
}
// WrapDirectory wraps a fs.Directory to include the info
@ -274,7 +282,7 @@ func (f *Fs) GetUsedSpace() (int64, error) {
}
func (f *Fs) updateUsage() (err error) {
if do := f.Fs.Features().About; do == nil {
if do := f.RootFs.Features().About; do == nil {
return ErrUsageFieldNotSupported
}
done := false
@ -301,7 +309,7 @@ func (f *Fs) updateUsageCore(lock bool) error {
// Run in background, should not be cancelled by user
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
usage, err := f.Features().About(ctx)
usage, err := f.RootFs.Features().About(ctx)
if err != nil {
f.cacheUpdate = false
return err