From 68ec6a9f5bec455b94b92cbea250e63af88aed5e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 23 Apr 2016 21:46:52 +0100 Subject: [PATCH] Add a directory parameter to Fs.List() --- amazonclouddrive/amazonclouddrive.go | 4 +- amazonclouddrive/amazonclouddrive_test.go | 2 +- b2/b2.go | 28 ++++--- b2/b2_test.go | 2 +- dircache/dircache.go | 11 +-- dircache/list.go | 16 +++- drive/drive.go | 4 +- drive/drive_test.go | 2 +- dropbox/dropbox.go | 16 +++- dropbox/dropbox_test.go | 2 +- dropbox/nametree.go | 24 +++++- dropbox/nametree_test.go | 73 +++++++++++-------- fs/fs.go | 15 ++-- fs/limited.go | 6 +- fs/operations.go | 32 ++++---- fs/operations_test.go | 4 +- fstest/fstest.go | 2 +- fstest/fstests/fstests.go | 67 ++++++----------- googlecloudstorage/googlecloudstorage.go | 30 +++++--- googlecloudstorage/googlecloudstorage_test.go | 2 +- hubic/hubic_test.go | 2 +- local/local.go | 54 +------------- local/local_test.go | 2 +- onedrive/onedrive.go | 4 +- onedrive/onedrive_test.go | 2 +- s3/s3.go | 26 +++++-- s3/s3_test.go | 2 +- swift/swift.go | 37 ++++++---- swift/swift_test.go | 2 +- yandex/yandex.go | 18 +++-- yandex/yandex_test.go | 2 +- 31 files changed, 263 insertions(+), 230 deletions(-) diff --git a/amazonclouddrive/amazonclouddrive.go b/amazonclouddrive/amazonclouddrive.go index f331e17b5..0345c0d76 100644 --- a/amazonclouddrive/amazonclouddrive.go +++ b/amazonclouddrive/amazonclouddrive.go @@ -375,8 +375,8 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache. } // List walks the path returning iles and directories into out -func (f *Fs) List(out fs.ListOpts) { - f.dirCache.List(f, out) +func (f *Fs) List(out fs.ListOpts, dir string) { + f.dirCache.List(f, out, dir) } // Put the object into the container diff --git a/amazonclouddrive/amazonclouddrive_test.go b/amazonclouddrive/amazonclouddrive_test.go index 2eab8b1e2..ff9be5034 100644 --- a/amazonclouddrive/amazonclouddrive_test.go +++ b/amazonclouddrive/amazonclouddrive_test.go @@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/b2/b2.go b/b2/b2.go index e5c56f1ba..4b71859cb 100644 --- a/b2/b2.go +++ b/b2/b2.go @@ -348,7 +348,11 @@ var errEndList = errors.New("end list") // than 1000) // // If hidden is set then it will list the hidden (deleted) files too. -func (f *Fs) list(level int, prefix string, limit int, hidden bool, fn listFn) error { +func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool, fn listFn) error { + root := f.root + if dir != "" { + root += dir + "/" + } bucketID, err := f.getBucketID() if err != nil { return err @@ -361,7 +365,7 @@ func (f *Fs) list(level int, prefix string, limit int, hidden bool, fn listFn) e BucketID: bucketID, MaxFileCount: chunkSize, } - prefix = f.root + prefix + prefix = root + prefix if prefix != "" { request.StartFileName = prefix } @@ -431,10 +435,10 @@ func (f *Fs) list(level int, prefix string, limit int, hidden bool, fn listFn) e } // listFiles walks the path returning files and directories to out -func (f *Fs) listFiles(out fs.ListOpts) { +func (f *Fs) listFiles(out fs.ListOpts, dir string) { defer out.Finished() // List the objects - err := f.list(out.Level(), "", 0, false, func(remote string, object *api.File, isDirectory bool) error { + err := f.list(dir, out.Level(), "", 0, false, func(remote string, object *api.File, isDirectory bool) error { if isDirectory { dir := &fs.Dir{ Name: remote, @@ -459,8 +463,12 @@ func (f *Fs) listFiles(out fs.ListOpts) { } // listBuckets returns all the buckets to out -func (f *Fs) listBuckets(out fs.ListOpts) { +func (f *Fs) listBuckets(out fs.ListOpts, dir string) { defer out.Finished() + if dir != "" { + out.SetError(fs.ErrorListOnlyRoot) + return + } err := f.listBucketsToFn(func(bucket *api.Bucket) error { dir := &fs.Dir{ Name: bucket.Name, @@ -478,11 +486,11 @@ func (f *Fs) listBuckets(out fs.ListOpts) { } // List walks the path returning files and directories to out -func (f *Fs) List(out fs.ListOpts) { +func (f *Fs) List(out fs.ListOpts, dir string) { if f.bucket == "" { - f.listBuckets(out) + f.listBuckets(out, dir) } else { - f.listFiles(out) + f.listFiles(out, dir) } return } @@ -678,7 +686,7 @@ func (f *Fs) Purge() error { } }() } - checkErr(f.list(fs.MaxLevel, "", 0, true, func(remote string, object *api.File, isDirectory bool) error { + checkErr(f.list("", fs.MaxLevel, "", 0, true, func(remote string, object *api.File, isDirectory bool) error { if !isDirectory { fs.Debug(remote, "Deleting (id %q)", object.ID) toBeDeleted <- object @@ -765,7 +773,7 @@ func (o *Object) readMetaData() (err error) { return nil } var info *api.File - err = o.fs.list(fs.MaxLevel, o.remote, 1, false, func(remote string, object *api.File, isDirectory bool) error { + err = o.fs.list("", fs.MaxLevel, o.remote, 1, false, func(remote string, object *api.File, isDirectory bool) error { if isDirectory { return nil } diff --git a/b2/b2_test.go b/b2/b2_test.go index 33a81b1b2..25a177105 100644 --- a/b2/b2_test.go +++ b/b2/b2_test.go @@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/dircache/dircache.go b/dircache/dircache.go index 4b7f8f1b3..d33026983 100644 --- a/dircache/dircache.go +++ b/dircache/dircache.go @@ -8,6 +8,8 @@ import ( "log" "strings" "sync" + + "github.com/ncw/rclone/fs" ) // DirCache caches paths to directory IDs and vice versa @@ -160,7 +162,7 @@ func (dc *DirCache) _findDir(path string, create bool) (pathID string, err error return "", fmt.Errorf("Failed to make directory: %v", err) } } else { - return "", fmt.Errorf("Couldn't find directory: %q", path) + return "", fs.ErrorDirNotFound } } @@ -179,13 +181,6 @@ func (dc *DirCache) FindPath(path string, create bool) (leaf, directoryID string defer dc.mu.Unlock() directory, leaf := SplitPath(path) directoryID, err = dc._findDir(directory, create) - if err != nil { - if create { - err = fmt.Errorf("Couldn't find or make directory %q: %s", directory, err) - } else { - err = fmt.Errorf("Couldn't find directory %q: %s", directory, err) - } - } return } diff --git a/dircache/list.go b/dircache/list.go index 4233abea0..66bbcc6d2 100644 --- a/dircache/list.go +++ b/dircache/list.go @@ -63,12 +63,20 @@ func listDir(f ListDirer, out fs.ListOpts, dirID string, path string) { } // List walks the path returning iles and directories into out -func (dc *DirCache) List(f ListDirer, out fs.ListOpts) { +func (dc *DirCache) List(f ListDirer, out fs.ListOpts, dir string) { defer out.Finished() err := dc.FindRoot(false) if err != nil { - out.SetError(fs.ErrorDirNotFound) - } else { - listDir(f, out, dc.RootID(), "") + out.SetError(err) + return } + id, err := dc.FindDir(dir, false) + if err != nil { + out.SetError(err) + return + } + if dir != "" { + dir += "/" + } + listDir(f, out, id, dir) } diff --git a/drive/drive.go b/drive/drive.go index f99637803..beb0addad 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -507,8 +507,8 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache. } // List walks the path returning files and directories to out -func (f *Fs) List(out fs.ListOpts) { - f.dirCache.List(f, out) +func (f *Fs) List(out fs.ListOpts, dir string) { + f.dirCache.List(f, out, dir) } // Creates a drive.File info from the parameters passed in and a half diff --git a/drive/drive_test.go b/drive/drive_test.go index 2af9b50e4..1d91af6f3 100644 --- a/drive/drive_test.go +++ b/drive/drive_test.go @@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go index 42afb7352..b5e53de4a 100644 --- a/dropbox/dropbox.go +++ b/dropbox/dropbox.go @@ -235,14 +235,22 @@ func (f *Fs) stripRoot(path string) (string, error) { } // Walk the root returning a channel of FsObjects -func (f *Fs) list(out fs.ListOpts) { +func (f *Fs) list(out fs.ListOpts, dir string) { // Track path component case, it could be different for entries coming from DropBox API // See https://www.dropboxforum.com/hc/communities/public/questions/201665409-Wrong-character-case-of-folder-name-when-calling-listFolder-using-Sync-API?locale=en-us // and https://github.com/ncw/rclone/issues/53 nameTree := newNameTree() cursor := "" + root := f.slashRoot + if dir != "" { + root += "/" + dir + // We assume that dir is entered in the correct case + // here which is likely since it probably came from a + // directory listing + nameTree.PutCaseCorrectPath(strings.Trim(root, "/")) + } for { - deltaPage, err := f.db.Delta(cursor, f.slashRoot) + deltaPage, err := f.db.Delta(cursor, root) if err != nil { fs.Stats.Error() fs.ErrorLog(f, "Couldn't list: %s", err) @@ -340,9 +348,9 @@ func (f *Fs) list(out fs.ListOpts) { } // List walks the path returning a channel of FsObjects -func (f *Fs) List(out fs.ListOpts) { +func (f *Fs) List(out fs.ListOpts, dir string) { defer out.Finished() - f.list(out) + f.list(out, dir) } // ListDir walks the path returning a channel of FsObjects diff --git a/dropbox/dropbox_test.go b/dropbox/dropbox_test.go index 0868fac07..deea5cf76 100644 --- a/dropbox/dropbox_test.go +++ b/dropbox/dropbox_test.go @@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/dropbox/nametree.go b/dropbox/nametree.go index c3e096bf0..3edace20e 100644 --- a/dropbox/nametree.go +++ b/dropbox/nametree.go @@ -46,7 +46,6 @@ func (tree *nameTreeNode) getTreeNode(path string) *nameTreeNode { // no lookup required, just return root return tree } - current := tree for _, component := range strings.Split(path, "/") { if len(component) == 0 { @@ -69,6 +68,29 @@ func (tree *nameTreeNode) getTreeNode(path string) *nameTreeNode { return current } +// PutCaseCorrectPath puts a known good path into the nameTree +func (tree *nameTreeNode) PutCaseCorrectPath(caseCorrectPath string) { + if len(caseCorrectPath) == 0 { + return + } + current := tree + for _, component := range strings.Split(caseCorrectPath, "/") { + if len(component) == 0 { + fs.Stats.Error() + fs.ErrorLog(tree, "PutCaseCorrectPath: path component is empty (full path %q)", caseCorrectPath) + return + } + lowercase := strings.ToLower(component) + lookup := current.Directories[lowercase] + if lookup == nil { + lookup = newNameTreeNode(component) + current.Directories[lowercase] = lookup + } + current = lookup + } + return +} + func (tree *nameTreeNode) PutCaseCorrectDirectoryName(parentPath string, caseCorrectDirectoryName string) { if len(caseCorrectDirectoryName) == 0 { fs.Stats.Error() diff --git a/dropbox/nametree_test.go b/dropbox/nametree_test.go index a30d99b6b..4719a61a9 100644 --- a/dropbox/nametree_test.go +++ b/dropbox/nametree_test.go @@ -5,32 +5,47 @@ import ( "github.com/ncw/rclone/fs" dropboxapi "github.com/stacktic/dropbox" + "github.com/stretchr/testify/assert" ) -func assert(t *testing.T, shouldBeTrue bool, failMessage string) { - if !shouldBeTrue { - t.Fatal(failMessage) - } -} - func TestPutCaseCorrectDirectoryName(t *testing.T) { errors := fs.Stats.GetErrors() tree := newNameTree() tree.PutCaseCorrectDirectoryName("a/b", "C") - assert(t, tree.CaseCorrectName == "", "Root CaseCorrectName should be empty") + assert.Equal(t, "", tree.CaseCorrectName, "Root CaseCorrectName should be empty") a := tree.Directories["a"] - assert(t, a.CaseCorrectName == "", "CaseCorrectName at 'a' should be empty") + assert.Equal(t, "", a.CaseCorrectName, "CaseCorrectName at 'a' should be empty") b := a.Directories["b"] - assert(t, b.CaseCorrectName == "", "CaseCorrectName at 'a/b' should be empty") + assert.Equal(t, "", b.CaseCorrectName, "CaseCorrectName at 'a/b' should be empty") c := b.Directories["c"] - assert(t, c.CaseCorrectName == "C", "CaseCorrectName at 'a/b/c' should be 'C'") + assert.Equal(t, "C", c.CaseCorrectName, "CaseCorrectName at 'a/b/c' should be 'C'") - assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported") + assert.Equal(t, errors, fs.Stats.GetErrors(), "No errors should be reported") +} + +func TestPutCaseCorrectPath(t *testing.T) { + errors := fs.Stats.GetErrors() + + tree := newNameTree() + tree.PutCaseCorrectPath("A/b/C") + + assert.Equal(t, "", tree.CaseCorrectName, "Root CaseCorrectName should be empty") + + a := tree.Directories["a"] + assert.Equal(t, "A", a.CaseCorrectName, "CaseCorrectName at 'a' should be 'A'") + + b := a.Directories["b"] + assert.Equal(t, "b", b.CaseCorrectName, "CaseCorrectName at 'a/b' should be 'b'") + + c := b.Directories["c"] + assert.Equal(t, "C", c.CaseCorrectName, "CaseCorrectName at 'a/b/c' should be 'C'") + + assert.Equal(t, errors, fs.Stats.GetErrors(), "No errors should be reported") } func TestPutCaseCorrectDirectoryNameEmptyComponent(t *testing.T) { @@ -41,7 +56,7 @@ func TestPutCaseCorrectDirectoryNameEmptyComponent(t *testing.T) { tree.PutCaseCorrectDirectoryName("b/", "C") tree.PutCaseCorrectDirectoryName("a//b", "C") - assert(t, fs.Stats.GetErrors() == errors+3, "3 errors should be reported") + assert.True(t, fs.Stats.GetErrors() == errors+3, "3 errors should be reported") } func TestPutCaseCorrectDirectoryNameEmptyParent(t *testing.T) { @@ -51,9 +66,9 @@ func TestPutCaseCorrectDirectoryNameEmptyParent(t *testing.T) { tree.PutCaseCorrectDirectoryName("", "C") c := tree.Directories["c"] - assert(t, c.CaseCorrectName == "C", "CaseCorrectName at 'c' should be 'C'") + assert.True(t, c.CaseCorrectName == "C", "CaseCorrectName at 'c' should be 'C'") - assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported") + assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported") } func TestGetPathWithCorrectCase(t *testing.T) { @@ -61,12 +76,12 @@ func TestGetPathWithCorrectCase(t *testing.T) { tree := newNameTree() tree.PutCaseCorrectDirectoryName("a", "C") - assert(t, tree.GetPathWithCorrectCase("a/c") == nil, "Path for 'a' should not be available") + assert.True(t, tree.GetPathWithCorrectCase("a/c") == nil, "Path for 'a' should not be available") tree.PutCaseCorrectDirectoryName("", "A") - assert(t, *tree.GetPathWithCorrectCase("a/c") == "/A/C", "Path for 'a/c' should be '/A/C'") + assert.True(t, *tree.GetPathWithCorrectCase("a/c") == "/A/C", "Path for 'a/c' should be '/A/C'") - assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported") + assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported") } func TestPutAndWalk(t *testing.T) { @@ -78,15 +93,15 @@ func TestPutAndWalk(t *testing.T) { numCalled := 0 walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) error { - assert(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath) - assert(t, entry.Path == "xxx", "entry.Path should be xxx") + assert.True(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath) + assert.True(t, entry.Path == "xxx", "entry.Path should be xxx") numCalled++ return nil } err := tree.WalkFiles("", walkFunc) - assert(t, err == nil, "No error should be returned") - assert(t, numCalled == 1, "walk func should be called only once") - assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported") + assert.True(t, err == nil, "No error should be returned") + assert.True(t, numCalled == 1, "walk func should be called only once") + assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported") } func TestPutAndWalkWithPrefix(t *testing.T) { @@ -98,15 +113,15 @@ func TestPutAndWalkWithPrefix(t *testing.T) { numCalled := 0 walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) error { - assert(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath) - assert(t, entry.Path == "xxx", "entry.Path should be xxx") + assert.True(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath) + assert.True(t, entry.Path == "xxx", "entry.Path should be xxx") numCalled++ return nil } err := tree.WalkFiles("A", walkFunc) - assert(t, err == nil, "No error should be returned") - assert(t, numCalled == 1, "walk func should be called only once") - assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported") + assert.True(t, err == nil, "No error should be returned") + assert.True(t, numCalled == 1, "walk func should be called only once") + assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported") } func TestPutAndWalkIncompleteTree(t *testing.T) { @@ -120,6 +135,6 @@ func TestPutAndWalkIncompleteTree(t *testing.T) { return nil } err := tree.WalkFiles("", walkFunc) - assert(t, err == nil, "No error should be returned") - assert(t, fs.Stats.GetErrors() == errors+1, "One error should be reported") + assert.True(t, err == nil, "No error should be returned") + assert.True(t, fs.Stats.GetErrors() == errors+1, "One error should be reported") } diff --git a/fs/fs.go b/fs/fs.go index 1969e3f06..9e5ca4279 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -40,6 +40,7 @@ var ( ErrorDirNotFound = fmt.Errorf("Directory not found") ErrorLevelNotSupported = fmt.Errorf("Level value not supported") ErrorListAborted = fmt.Errorf("List aborted") + ErrorListOnlyRoot = fmt.Errorf("Can only list from root") ) // RegInfo provides information about a filesystem @@ -98,10 +99,14 @@ func Register(info *RegInfo) { type Fs interface { Info - // List the objects and directories of the Fs + // List the objects and directories of the Fs starting from dir // - // This should return ErrDirNotFound if the directory isn't found. - List(ListOpts) + // dir should be "" to start from the root, and should not + // have trailing slashes. + // + // This should return ErrDirNotFound (using out.SetError()) + // if the directory isn't found. + List(out ListOpts, dir string) // NewFsObject finds the Object at remote. Returns nil if can't be found NewFsObject(remote string) Object @@ -312,10 +317,10 @@ func NewLister() *Lister { // Start starts a go routine listing the Fs passed in. It returns the // same Lister that was passed in for convenience. -func (o *Lister) Start(f Fs) *Lister { +func (o *Lister) Start(f Fs, dir string) *Lister { o.results = make(chan listerResult, o.buffer) go func() { - f.List(o) + f.List(o, dir) }() return o } diff --git a/fs/limited.go b/fs/limited.go index 2139ba6d1..bda083699 100644 --- a/fs/limited.go +++ b/fs/limited.go @@ -38,8 +38,12 @@ func (f *Limited) String() string { } // List the Fs into a channel -func (f *Limited) List(opts ListOpts) { +func (f *Limited) List(opts ListOpts, dir string) { defer opts.Finished() + if dir != "" { + opts.SetError(ErrorListOnlyRoot) + return + } for _, obj := range f.objects { if opts.Add(obj) { return diff --git a/fs/operations.go b/fs/operations.go index c8459eccb..8c9568766 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -454,16 +454,17 @@ func DeleteFiles(toBeDeleted ObjectsChan) { } // Read a map of Object.Remote to Object for the given Fs. +// dir is the start directory, "" for root // If includeAll is specified all files will be added, // otherwise only files passing the filter will be added. -func readFilesMap(fs Fs, includeAll bool) (files map[string]Object, err error) { +func readFilesMap(fs Fs, includeAll bool, dir string) (files map[string]Object, err error) { files = make(map[string]Object) normalised := make(map[string]struct{}) list := NewLister() if !includeAll { list.SetFilter(Config.Filter) } - list.Start(fs) + list.Start(fs, dir) for { o, err := list.GetObject() if err != nil { @@ -494,14 +495,15 @@ func readFilesMap(fs Fs, includeAll bool) (files map[string]Object, err error) { } // readFilesMaps runs readFilesMap on fdst and fsrc at the same time -func readFilesMaps(fdst Fs, fdstIncludeAll bool, fsrc Fs, fsrcIncludeAll bool) (dstFiles, srcFiles map[string]Object, err error) { +// dir is the start directory, "" for root +func readFilesMaps(fdst Fs, fdstIncludeAll bool, fsrc Fs, fsrcIncludeAll bool, dir string) (dstFiles, srcFiles map[string]Object, err error) { var wg sync.WaitGroup var srcErr, dstErr error list := func(fs Fs, includeAll bool, pMap *map[string]Object, pErr *error) { defer wg.Done() Log(fs, "Building file list") - dstFiles, listErr := readFilesMap(fs, includeAll) + dstFiles, listErr := readFilesMap(fs, includeAll, dir) if listErr != nil { ErrorLog(fs, "Error building file list: %v", listErr) *pErr = listErr @@ -535,7 +537,9 @@ func Same(fdst, fsrc Fs) bool { // If Delete is true then it deletes any files in fdst that aren't in fsrc // // If DoMove is true then files will be moved instead of copied -func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error { +// +// dir is the start directory, "" for root +func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool, dir string) error { if Same(fdst, fsrc) { ErrorLog(fdst, "Nothing to do as source and destination are the same") return nil @@ -547,7 +551,7 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error { } // Read the files of both source and destination in parallel - dstFiles, srcFiles, err := readFilesMaps(fdst, Config.Filter.DeleteExcluded, fsrc, false) + dstFiles, srcFiles, err := readFilesMaps(fdst, Config.Filter.DeleteExcluded, fsrc, false, dir) if err != nil { return err } @@ -651,12 +655,12 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error { // Sync fsrc into fdst func Sync(fdst, fsrc Fs) error { - return syncCopyMove(fdst, fsrc, true, false) + return syncCopyMove(fdst, fsrc, true, false, "") } // CopyDir copies fsrc into fdst func CopyDir(fdst, fsrc Fs) error { - return syncCopyMove(fdst, fsrc, false, false) + return syncCopyMove(fdst, fsrc, false, false, "") } // MoveDir moves fsrc into fdst @@ -684,7 +688,7 @@ func MoveDir(fdst, fsrc Fs) error { } // Now move the files - err := syncCopyMove(fdst, fsrc, false, true) + err := syncCopyMove(fdst, fsrc, false, true, "") if err != nil || Stats.Errored() { ErrorLog(fdst, "Not deleting files as there were IO errors") return err @@ -732,7 +736,7 @@ func checkIdentical(dst, src Object) bool { // Check the files in fsrc and fdst according to Size and hash func Check(fdst, fsrc Fs) error { - dstFiles, srcFiles, err := readFilesMaps(fdst, false, fsrc, false) + dstFiles, srcFiles, err := readFilesMaps(fdst, false, fsrc, false, "") if err != nil { return err } @@ -800,7 +804,7 @@ func Check(fdst, fsrc Fs) error { // // Lists in parallel which may get them out of order func ListFn(f Fs, fn func(Object)) error { - list := NewLister().SetFilter(Config.Filter).Start(f) + list := NewLister().SetFilter(Config.Filter).Start(f, "") var wg sync.WaitGroup wg.Add(Config.Checkers) for i := 0; i < Config.Checkers; i++ { @@ -909,7 +913,7 @@ func Count(f Fs) (objects int64, size int64, err error) { // ListDir lists the directories/buckets/containers in the Fs to the supplied writer func ListDir(f Fs, w io.Writer) error { - list := NewLister().SetLevel(1).Start(f) + list := NewLister().SetLevel(1).Start(f, "") for { dir, err := list.GetDir() if err != nil { @@ -976,7 +980,7 @@ func Purge(f Fs) error { } if doFallbackPurge { // DeleteFiles and Rmdir observe --dry-run - list := NewLister().Start(f) + list := NewLister().Start(f, "") DeleteFiles(listToChan(list)) err = Rmdir(f) } @@ -1132,7 +1136,7 @@ func (mode DeduplicateMode) String() string { func Deduplicate(f Fs, mode DeduplicateMode) error { Log(f, "Looking for duplicates using %v mode.", mode) files := map[string][]Object{} - list := NewLister().Start(f) + list := NewLister().Start(f, "") for { o, err := list.GetObject() if err != nil { diff --git a/fs/operations_test.go b/fs/operations_test.go index 19c36e94e..8900f1d1b 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -140,7 +140,7 @@ func NewRun(t *testing.T) *Run { r = new(Run) *r = *oneRun r.cleanRemote = func() { - list := fs.NewLister().Start(r.fremote) + list := fs.NewLister().Start(r.fremote, "") for { o, err := list.GetObject() if err != nil { @@ -1181,7 +1181,7 @@ func TestDeduplicateRename(t *testing.T) { t.Fatalf("fs.Deduplicate returned error: %v", err) } - list := fs.NewLister().Start(r.fremote) + list := fs.NewLister().Start(r.fremote, "") for { o, err := list.GetObject() if err != nil { diff --git a/fstest/fstest.go b/fstest/fstest.go index 89b674daa..97e48b2fd 100644 --- a/fstest/fstest.go +++ b/fstest/fstest.go @@ -158,7 +158,7 @@ func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision ti const retries = 6 sleep := time.Second / 2 for i := 1; i <= retries; i++ { - objs, err = fs.NewLister().Start(f).GetObjects() + objs, err = fs.NewLister().Start(f, "").GetObjects() if err != nil && err != fs.ErrorDirNotFound { t.Fatalf("Error listing: %v", err) } diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index fd8a418ea..3f0c520c4 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -11,12 +11,15 @@ import ( "io" "log" "os" + "path" "strings" "testing" "time" "github.com/ncw/rclone/fs" "github.com/ncw/rclone/fstest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var ( @@ -129,10 +132,8 @@ func TestFsListEmpty(t *testing.T) { // TestFsListDirEmpty tests listing the directories from an empty directory func TestFsListDirEmpty(t *testing.T) { skipIfNotOk(t) - dirs, err := fs.NewLister().SetLevel(1).Start(remote).GetDirs() - if err != nil { - t.Fatal(err) - } + dirs, err := fs.NewLister().SetLevel(1).Start(remote, "").GetDirs() + require.NoError(t, err) for _, dir := range dirs { t.Errorf("Found unexpected item %q", dir.Name) } @@ -197,10 +198,8 @@ func TestFsListDirFile2(t *testing.T) { skipIfNotOk(t) found := false for i := 1; i <= eventualConsistencyRetries; i++ { - dirs, err := fs.NewLister().SetLevel(1).Start(remote).GetDirs() - if err != nil { - t.Fatal(err) - } + dirs, err := fs.NewLister().SetLevel(1).Start(remote, "").GetDirs() + require.NoError(t, err) for _, dir := range dirs { if dir.Name != `hello? sausage` && dir.Name != `hello_ sausage` { t.Errorf("Found unexpected item %q", dir.Name) @@ -227,10 +226,8 @@ func TestFsListDirRoot(t *testing.T) { t.Fatalf("Failed to make remote %q: %v", RemoteName, err) } found := false - dirs, err := fs.NewLister().SetLevel(1).Start(rootRemote).GetDirs() - if err != nil { - t.Fatal(err) - } + dirs, err := fs.NewLister().SetLevel(1).Start(rootRemote, "").GetDirs() + require.NoError(t, err) for _, dir := range dirs { if dir.Name == subRemoteLeaf { found = true @@ -241,42 +238,22 @@ func TestFsListDirRoot(t *testing.T) { } } -// TestFsListRoot tests List works in the root -func TestFsListRoot(t *testing.T) { +// TestFsListSubdir tests List works for a subdirectory +func TestFsListSubdir(t *testing.T) { skipIfNotOk(t) - rootRemote, err := fs.NewFs(RemoteName) - if err != nil { - t.Fatalf("Failed to make remote %q: %v", RemoteName, err) - } - // Should either find file1 and file2 or nothing - found1 := false - f1 := subRemoteLeaf + "/" + file1.Path - found2 := false - f2 := subRemoteLeaf + "/" + file2.Path - f2Alt := subRemoteLeaf + "/" + file2.WinPath - count := 0 - objs, err := fs.NewLister().Start(rootRemote).GetObjects() - if err != nil { - t.Fatal(err) - } - for _, obj := range objs { - count++ - if obj.Remote() == f1 { - found1 = true - } - if obj.Remote() == f2 || obj.Remote() == f2Alt { - found2 = true + test := func(fileName string) bool { + dir, _ := path.Split(fileName) + dir = dir[:len(dir)-1] + objs, err := fs.NewLister().Start(remote, dir).GetObjects() + if err == fs.ErrorDirNotFound { + return false } + require.NoError(t, err) + require.Len(t, objs, 1) + assert.Equal(t, fileName, objs[0].Remote()) + return true } - if count == 0 { - // Nothing found is OK - return - } - if found1 && found2 { - // Both found is OK - return - } - t.Errorf("Didn't find %q (%v) and %q (%v) or no files (count %d)", f1, found1, f2, found2, count) + assert.True(t, test(file2.Path) || test(file2.WinPath), "normal and alternative lists failed") } // TestFsListFile1 tests file present diff --git a/googlecloudstorage/googlecloudstorage.go b/googlecloudstorage/googlecloudstorage.go index bc5554863..90074c9c6 100644 --- a/googlecloudstorage/googlecloudstorage.go +++ b/googlecloudstorage/googlecloudstorage.go @@ -300,9 +300,15 @@ type listFn func(remote string, object *storage.Object, isDirectory bool) error // list the objects into the function supplied // +// dir is the starting directory, "" for root +// // If directories is set it only sends directories -func (f *Fs) list(level int, fn listFn) error { - list := f.svc.Objects.List(f.bucket).Prefix(f.root).MaxResults(listChunks) +func (f *Fs) list(dir string, level int, fn listFn) error { + root := f.root + if dir != "" { + root += dir + "/" + } + list := f.svc.Objects.List(f.bucket).Prefix(root).MaxResults(listChunks) switch level { case 1: list = list.Delimiter("/") @@ -310,7 +316,7 @@ func (f *Fs) list(level int, fn listFn) error { default: return fs.ErrorLevelNotSupported } - rootLength := len(f.root) + rootLength := len(root) for { objects, err := list.Do() if err != nil { @@ -329,7 +335,7 @@ func (f *Fs) list(level int, fn listFn) error { } } for _, object := range objects.Items { - if !strings.HasPrefix(object.Name, f.root) { + if !strings.HasPrefix(object.Name, root) { fs.Log(f, "Odd name received %q", object.Name) continue } @@ -348,14 +354,14 @@ func (f *Fs) list(level int, fn listFn) error { } // listFiles lists files and directories to out -func (f *Fs) listFiles(out fs.ListOpts) { +func (f *Fs) listFiles(out fs.ListOpts, dir string) { defer out.Finished() if f.bucket == "" { out.SetError(fmt.Errorf("Can't list objects at root - choose a bucket using lsd")) return } // List the objects - err := f.list(out.Level(), func(remote string, object *storage.Object, isDirectory bool) error { + err := f.list(dir, out.Level(), func(remote string, object *storage.Object, isDirectory bool) error { if isDirectory { dir := &fs.Dir{ Name: remote, @@ -385,8 +391,12 @@ func (f *Fs) listFiles(out fs.ListOpts) { } // listBuckets lists the buckets to out -func (f *Fs) listBuckets(out fs.ListOpts) { +func (f *Fs) listBuckets(out fs.ListOpts, dir string) { defer out.Finished() + if dir != "" { + out.SetError(fs.ErrorListOnlyRoot) + return + } if f.projectNumber == "" { out.SetError(errors.New("Can't list buckets without project number")) return @@ -416,11 +426,11 @@ func (f *Fs) listBuckets(out fs.ListOpts) { } // List lists the path to out -func (f *Fs) List(out fs.ListOpts) { +func (f *Fs) List(out fs.ListOpts, dir string) { if f.bucket == "" { - f.listBuckets(out) + f.listBuckets(out, dir) } else { - f.listFiles(out) + f.listFiles(out, dir) } return } diff --git a/googlecloudstorage/googlecloudstorage_test.go b/googlecloudstorage/googlecloudstorage_test.go index 59e20e843..0cb81a5b9 100644 --- a/googlecloudstorage/googlecloudstorage_test.go +++ b/googlecloudstorage/googlecloudstorage_test.go @@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/hubic/hubic_test.go b/hubic/hubic_test.go index e91842159..9a615ee17 100644 --- a/hubic/hubic_test.go +++ b/hubic/hubic_test.go @@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/local/local.go b/local/local.go index 51e1c2d2f..b9c0b7321 100644 --- a/local/local.go +++ b/local/local.go @@ -210,13 +210,12 @@ func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (su // List the path into out // // Ignores everything which isn't Storable, eg links etc -func (f *Fs) List(out fs.ListOpts) { +func (f *Fs) List(out fs.ListOpts, dir string) { defer out.Finished() - _, err := os.Stat(f.root) + root := path.Join(f.root, dir) + _, err := os.Stat(root) if err != nil { out.SetError(fs.ErrorDirNotFound) - fs.Stats.Error() - fs.ErrorLog(f, "Directory not found: %s: %s", f.root, err) return } @@ -226,7 +225,7 @@ func (f *Fs) List(out fs.ListOpts) { // Start the process traversing.Add(1) - in <- listArgs{remote: "", dirpath: f.root, level: out.Level() - 1} + in <- listArgs{remote: "", dirpath: root, level: out.Level() - 1} for i := 0; i < fs.Config.Checkers; i++ { wg.Add(1) go func() { @@ -276,51 +275,6 @@ func (f *Fs) cleanUtf8(name string) string { return name } -/* -// ListDir walks the path returning a channel of FsObjects -func (f *Fs) ListDir(out fs.ListDirOpts) { - defer out.Finished() - items, err := ioutil.ReadDir(f.root) - if err != nil { - fs.Stats.Error() - fs.ErrorLog(f, "Couldn't find read directory: %s", err) - out.SetError(err) - return - } - for _, item := range items { - if item.IsDir() { - dir := &fs.Dir{ - Name: f.cleanUtf8(item.Name()), - When: item.ModTime(), - Bytes: 0, - Count: 0, - } - // Go down the tree to count the files and directories - dirpath := f.filterPath(filepath.Join(f.root, item.Name())) - err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error { - if err != nil { - fs.Stats.Error() - fs.ErrorLog(f, "Failed to open directory: %s: %s", path, err) - out.SetError(err) - } else { - dir.Count++ - dir.Bytes += fi.Size() - } - return nil - }) - if err != nil { - out.SetError(err) - fs.Stats.Error() - fs.ErrorLog(f, "Failed to open directory: %s: %s", dirpath, err) - } - if out.Add(dir) { - return - } - } - } -} -*/ - // Put the FsObject to the local filesystem func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { remote := src.Remote() diff --git a/local/local_test.go b/local/local_test.go index 22a2e5279..6af4b578e 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/onedrive/onedrive.go b/onedrive/onedrive.go index 7ec0cc99f..1c3d44b95 100644 --- a/onedrive/onedrive.go +++ b/onedrive/onedrive.go @@ -405,8 +405,8 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache. } // List walks the path returning files and directories into out -func (f *Fs) List(out fs.ListOpts) { - f.dirCache.List(f, out) +func (f *Fs) List(out fs.ListOpts, dir string) { + f.dirCache.List(f, out, dir) } // Creates from the parameters passed in a half finished Object which diff --git a/onedrive/onedrive_test.go b/onedrive/onedrive_test.go index 73958ec08..b728e242b 100644 --- a/onedrive/onedrive_test.go +++ b/onedrive/onedrive_test.go @@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/s3/s3.go b/s3/s3.go index 5b456c323..6df5c7f29 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -369,8 +369,14 @@ type listFn func(remote string, object *s3.Object, isDirectory bool) error // list the objects into the function supplied // +// dir is the starting directory, "" for root +// // Level is the level of the recursion -func (f *Fs) list(level int, fn listFn) error { +func (f *Fs) list(dir string, level int, fn listFn) error { + root := f.root + if dir != "" { + root += dir + "/" + } maxKeys := int64(listChunkSize) delimiter := "" switch level { @@ -386,7 +392,7 @@ func (f *Fs) list(level int, fn listFn) error { req := s3.ListObjectsInput{ Bucket: &f.bucket, Delimiter: &delimiter, - Prefix: &f.root, + Prefix: &root, MaxKeys: &maxKeys, Marker: marker, } @@ -442,7 +448,7 @@ func (f *Fs) list(level int, fn listFn) error { } // listFiles lists files and directories to out -func (f *Fs) listFiles(out fs.ListOpts) { +func (f *Fs) listFiles(out fs.ListOpts, dir string) { defer out.Finished() if f.bucket == "" { // Return no objects at top level list @@ -450,7 +456,7 @@ func (f *Fs) listFiles(out fs.ListOpts) { return } // List the objects and directories - err := f.list(out.Level(), func(remote string, object *s3.Object, isDirectory bool) error { + err := f.list(dir, out.Level(), func(remote string, object *s3.Object, isDirectory bool) error { if isDirectory { size := int64(0) if object.Size != nil { @@ -484,8 +490,12 @@ func (f *Fs) listFiles(out fs.ListOpts) { } // listBuckets lists the buckets to out -func (f *Fs) listBuckets(out fs.ListOpts) { +func (f *Fs) listBuckets(out fs.ListOpts, dir string) { defer out.Finished() + if dir != "" { + out.SetError(fs.ErrorListOnlyRoot) + return + } req := s3.ListBucketsInput{} resp, err := f.c.ListBuckets(&req) if err != nil { @@ -506,11 +516,11 @@ func (f *Fs) listBuckets(out fs.ListOpts) { } // List lists files and directories to out -func (f *Fs) List(out fs.ListOpts) { +func (f *Fs) List(out fs.ListOpts, dir string) { if f.bucket == "" { - f.listBuckets(out) + f.listBuckets(out, dir) } else { - f.listFiles(out) + f.listFiles(out, dir) } return } diff --git a/s3/s3_test.go b/s3/s3_test.go index 6c4596ca5..add4d4abf 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/swift/swift.go b/swift/swift.go index 9d450eabd..e652eef2c 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -260,10 +260,14 @@ type listFn func(remote string, object *swift.Object, isDirectory bool) error // the container and root supplied // // Level is the level of the recursion -func (f *Fs) listContainerRoot(container, root string, level int, fn listFn) error { +func (f *Fs) listContainerRoot(container, root string, dir string, level int, fn listFn) error { + prefix := root + if dir != "" { + prefix += dir + "/" + } // Options for ObjectsWalk opts := swift.ObjectsOpts{ - Prefix: root, + Prefix: prefix, Limit: 256, } switch level { @@ -302,21 +306,19 @@ func (f *Fs) listContainerRoot(container, root string, level int, fn listFn) err } // list the objects into the function supplied -func (f *Fs) list(level int, fn listFn) error { - return f.listContainerRoot(f.container, f.root, level, fn) +func (f *Fs) list(dir string, level int, fn listFn) error { + return f.listContainerRoot(f.container, f.root, dir, level, fn) } // listFiles walks the path returning a channel of FsObjects -// -// if ignoreStorable is set then it outputs the file even if Storable() is false -func (f *Fs) listFiles(out fs.ListOpts, ignoreStorable bool) { +func (f *Fs) listFiles(out fs.ListOpts, dir string) { defer out.Finished() if f.container == "" { out.SetError(errors.New("Can't list objects at root - choose a container using lsd")) return } // List the objects - err := f.list(out.Level(), func(remote string, object *swift.Object, isDirectory bool) error { + err := f.list(dir, out.Level(), func(remote string, object *swift.Object, isDirectory bool) error { if isDirectory { dir := &fs.Dir{ Name: remote, @@ -329,8 +331,7 @@ func (f *Fs) listFiles(out fs.ListOpts, ignoreStorable bool) { } else { if o := f.newFsObjectWithInfo(remote, object); o != nil { // Storable does a full metadata read on 0 size objects which might be dynamic large objects - storable := o.Storable() - if storable || ignoreStorable { + if o.Storable() { if out.Add(o) { return fs.ErrorListAborted } @@ -348,8 +349,12 @@ func (f *Fs) listFiles(out fs.ListOpts, ignoreStorable bool) { } // listContainers lists the containers -func (f *Fs) listContainers(out fs.ListOpts) { +func (f *Fs) listContainers(out fs.ListOpts, dir string) { defer out.Finished() + if dir != "" { + out.SetError(fs.ErrorListOnlyRoot) + return + } containers, err := f.c.ContainersAll(nil) if err != nil { out.SetError(err) @@ -368,11 +373,11 @@ func (f *Fs) listContainers(out fs.ListOpts) { } // List walks the path returning files and directories to out -func (f *Fs) List(out fs.ListOpts) { +func (f *Fs) List(out fs.ListOpts, dir string) { if f.container == "" { - f.listContainers(out) + f.listContainers(out, dir) } else { - f.listFiles(out, false) + f.listFiles(out, dir) } return } @@ -428,7 +433,7 @@ func (f *Fs) Purge() error { toBeDeleted := make(chan fs.Object, fs.Config.Transfers) var err error go func() { - err = f.list(fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error { + err = f.list("", fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error { if !isDirectory { if o := f.newFsObjectWithInfo(remote, object); o != nil { toBeDeleted <- o @@ -625,7 +630,7 @@ func min(x, y int64) int64 { // if except is passed in then segments with that prefix won't be deleted func (o *Object) removeSegments(except string) error { segmentsRoot := o.fs.root + o.remote + "/" - err := o.fs.listContainerRoot(o.fs.segmentsContainer, segmentsRoot, fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error { + err := o.fs.listContainerRoot(o.fs.segmentsContainer, segmentsRoot, "", fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error { if isDirectory { return nil } diff --git a/swift/swift_test.go b/swift/swift_test.go index ec6a39498..bdaee6ab5 100644 --- a/swift/swift_test.go +++ b/swift/swift_test.go @@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/yandex/yandex.go b/yandex/yandex.go index 87e952200..0eef428d2 100644 --- a/yandex/yandex.go +++ b/yandex/yandex.go @@ -200,7 +200,7 @@ func (f *Fs) listDir(fn listFn) (err error) { // list the objects into the function supplied // // This does a flat listing of all the files in the drive -func (f *Fs) list(fn listFn) error { +func (f *Fs) list(dir string, fn listFn) error { //request files list. list is divided into pages. We send request for each page //items per page is limited by limit //TODO may be add config parameter for the items per page limit @@ -211,6 +211,10 @@ func (f *Fs) list(fn listFn) error { var opt yandex.FlatFileListRequestOptions opt.Limit = &limit opt.Offset = &offset + prefix := f.diskRoot + if dir != "" { + prefix += dir + "/" + } //query each page of list until itemCount is less then limit for { //send request @@ -223,7 +227,7 @@ func (f *Fs) list(fn listFn) error { //list files for _, item := range info.Items { // filter file list and get only files we need - if strings.HasPrefix(item.Path, f.diskRoot) { + if strings.HasPrefix(item.Path, prefix) { //trim root folder from filename var name = strings.TrimPrefix(item.Path, f.diskRoot) err = fn(name, &item, false) @@ -244,7 +248,7 @@ func (f *Fs) list(fn listFn) error { } // List walks the path returning a channel of FsObjects -func (f *Fs) List(out fs.ListOpts) { +func (f *Fs) List(out fs.ListOpts, dir string) { defer out.Finished() listItem := func(remote string, object *yandex.ResourceInfoResponse, isDirectory bool) error { @@ -275,9 +279,13 @@ func (f *Fs) List(out fs.ListOpts) { var err error switch out.Level() { case 1: - err = f.listDir(listItem) + if dir == "" { + err = f.listDir(listItem) + } else { + err = f.list(dir, listItem) + } case fs.MaxLevel: - err = f.list(listItem) + err = f.list(dir, listItem) default: out.SetError(fs.ErrorLevelNotSupported) } diff --git a/yandex/yandex_test.go b/yandex/yandex_test.go index 10c270a44..3e5ad58e2 100644 --- a/yandex/yandex_test.go +++ b/yandex/yandex_test.go @@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }