extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-01-03 04:49:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add a directory parameter to Fs.List()

Browse Source
This commit is contained in:
Nick Craig-Wood 2016-04-23 21:46:52 +01:00
parent 753b0717be
commit 68ec6a9f5b
31 changed files with 263 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
amazonclouddrive/amazonclouddrive.go
View File

@ -375,8 +375,8 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
}
// List walks the path returning iles and directories into out
func (f *Fs) List(out fs.ListOpts) {
f.dirCache.List(f, out)
func (f *Fs) List(out fs.ListOpts, dir string) {
f.dirCache.List(f, out, dir)
}
// Put the object into the container

2
amazonclouddrive/amazonclouddrive_test.go
View File

@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

28
b2/b2.go
View File

@ -348,7 +348,11 @@ var errEndList = errors.New("end list")
// than 1000)
//
// If hidden is set then it will list the hidden (deleted) files too.
func (f *Fs) list(level int, prefix string, limit int, hidden bool, fn listFn) error {
func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool, fn listFn) error {
root := f.root
if dir != "" {
root += dir + "/"
}
bucketID, err := f.getBucketID()
if err != nil {
return err
@ -361,7 +365,7 @@ func (f *Fs) list(level int, prefix string, limit int, hidden bool, fn listFn) e
BucketID: bucketID,
MaxFileCount: chunkSize,
}
prefix = f.root + prefix
prefix = root + prefix
if prefix != "" {
request.StartFileName = prefix
}
@ -431,10 +435,10 @@ func (f *Fs) list(level int, prefix string, limit int, hidden bool, fn listFn) e
}
// listFiles walks the path returning files and directories to out
func (f *Fs) listFiles(out fs.ListOpts) {
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
// List the objects
err := f.list(out.Level(), "", 0, false, func(remote string, object *api.File, isDirectory bool) error {
err := f.list(dir, out.Level(), "", 0, false, func(remote string, object *api.File, isDirectory bool) error {
if isDirectory {
dir := &fs.Dir{
Name: remote,
@ -459,8 +463,12 @@ func (f *Fs) listFiles(out fs.ListOpts) {
}
// listBuckets returns all the buckets to out
func (f *Fs) listBuckets(out fs.ListOpts) {
func (f *Fs) listBuckets(out fs.ListOpts, dir string) {
defer out.Finished()
if dir != "" {
out.SetError(fs.ErrorListOnlyRoot)
return
}
err := f.listBucketsToFn(func(bucket *api.Bucket) error {
dir := &fs.Dir{
Name: bucket.Name,
@ -478,11 +486,11 @@ func (f *Fs) listBuckets(out fs.ListOpts) {
}
// List walks the path returning files and directories to out
func (f *Fs) List(out fs.ListOpts) {
func (f *Fs) List(out fs.ListOpts, dir string) {
if f.bucket == "" {
f.listBuckets(out)
f.listBuckets(out, dir)
} else {
f.listFiles(out)
f.listFiles(out, dir)
}
return
}
@ -678,7 +686,7 @@ func (f *Fs) Purge() error {
}
}()
}
checkErr(f.list(fs.MaxLevel, "", 0, true, func(remote string, object *api.File, isDirectory bool) error {
checkErr(f.list("", fs.MaxLevel, "", 0, true, func(remote string, object *api.File, isDirectory bool) error {
if !isDirectory {
fs.Debug(remote, "Deleting (id %q)", object.ID)
toBeDeleted <- object
@ -765,7 +773,7 @@ func (o *Object) readMetaData() (err error) {
return nil
}
var info *api.File
err = o.fs.list(fs.MaxLevel, o.remote, 1, false, func(remote string, object *api.File, isDirectory bool) error {
err = o.fs.list("", fs.MaxLevel, o.remote, 1, false, func(remote string, object *api.File, isDirectory bool) error {
if isDirectory {
return nil
}

2
b2/b2_test.go
View File

@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

11
dircache/dircache.go
View File

@ -8,6 +8,8 @@ import (
"log"
"strings"
"sync"
"github.com/ncw/rclone/fs"
)
// DirCache caches paths to directory IDs and vice versa
@ -160,7 +162,7 @@ func (dc *DirCache) _findDir(path string, create bool) (pathID string, err error
return "", fmt.Errorf("Failed to make directory: %v", err)
}
} else {
return "", fmt.Errorf("Couldn't find directory: %q", path)
return "", fs.ErrorDirNotFound
}
}
@ -179,13 +181,6 @@ func (dc *DirCache) FindPath(path string, create bool) (leaf, directoryID string
defer dc.mu.Unlock()
directory, leaf := SplitPath(path)
directoryID, err = dc._findDir(directory, create)
if err != nil {
if create {
err = fmt.Errorf("Couldn't find or make directory %q: %s", directory, err)
} else {
err = fmt.Errorf("Couldn't find directory %q: %s", directory, err)
}
}
return
}

16
dircache/list.go
View File

@ -63,12 +63,20 @@ func listDir(f ListDirer, out fs.ListOpts, dirID string, path string) {
}
// List walks the path returning iles and directories into out
func (dc *DirCache) List(f ListDirer, out fs.ListOpts) {
func (dc *DirCache) List(f ListDirer, out fs.ListOpts, dir string) {
defer out.Finished()
err := dc.FindRoot(false)
if err != nil {
out.SetError(fs.ErrorDirNotFound)
} else {
listDir(f, out, dc.RootID(), "")
out.SetError(err)
return
}
id, err := dc.FindDir(dir, false)
if err != nil {
out.SetError(err)
return
}
if dir != "" {
dir += "/"
}
listDir(f, out, id, dir)
}

4
drive/drive.go
View File

@ -507,8 +507,8 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
}
// List walks the path returning files and directories to out
func (f *Fs) List(out fs.ListOpts) {
f.dirCache.List(f, out)
func (f *Fs) List(out fs.ListOpts, dir string) {
f.dirCache.List(f, out, dir)
}
// Creates a drive.File info from the parameters passed in and a half

2
drive/drive_test.go
View File

@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

16
dropbox/dropbox.go
View File

@ -235,14 +235,22 @@ func (f *Fs) stripRoot(path string) (string, error) {
}
// Walk the root returning a channel of FsObjects
func (f *Fs) list(out fs.ListOpts) {
func (f *Fs) list(out fs.ListOpts, dir string) {
// Track path component case, it could be different for entries coming from DropBox API
// See https://www.dropboxforum.com/hc/communities/public/questions/201665409-Wrong-character-case-of-folder-name-when-calling-listFolder-using-Sync-API?locale=en-us
// and https://github.com/ncw/rclone/issues/53
nameTree := newNameTree()
cursor := ""
root := f.slashRoot
if dir != "" {
root += "/" + dir
// We assume that dir is entered in the correct case
// here which is likely since it probably came from a
// directory listing
nameTree.PutCaseCorrectPath(strings.Trim(root, "/"))
}
for {
deltaPage, err := f.db.Delta(cursor, f.slashRoot)
deltaPage, err := f.db.Delta(cursor, root)
if err != nil {
fs.Stats.Error()
fs.ErrorLog(f, "Couldn't list: %s", err)
@ -340,9 +348,9 @@ func (f *Fs) list(out fs.ListOpts) {
}
// List walks the path returning a channel of FsObjects
func (f *Fs) List(out fs.ListOpts) {
func (f *Fs) List(out fs.ListOpts, dir string) {
defer out.Finished()
f.list(out)
f.list(out, dir)
}
// ListDir walks the path returning a channel of FsObjects

2
dropbox/dropbox_test.go
View File

@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

24
dropbox/nametree.go
View File

@ -46,7 +46,6 @@ func (tree *nameTreeNode) getTreeNode(path string) *nameTreeNode {
// no lookup required, just return root
return tree
}
current := tree
for _, component := range strings.Split(path, "/") {
if len(component) == 0 {
@ -69,6 +68,29 @@ func (tree *nameTreeNode) getTreeNode(path string) *nameTreeNode {
return current
}
// PutCaseCorrectPath puts a known good path into the nameTree
func (tree *nameTreeNode) PutCaseCorrectPath(caseCorrectPath string) {
if len(caseCorrectPath) == 0 {
return
}
current := tree
for _, component := range strings.Split(caseCorrectPath, "/") {
if len(component) == 0 {
fs.Stats.Error()
fs.ErrorLog(tree, "PutCaseCorrectPath: path component is empty (full path %q)", caseCorrectPath)
return
}
lowercase := strings.ToLower(component)
lookup := current.Directories[lowercase]
if lookup == nil {
lookup = newNameTreeNode(component)
current.Directories[lowercase] = lookup
}
current = lookup
}
return
}
func (tree *nameTreeNode) PutCaseCorrectDirectoryName(parentPath string, caseCorrectDirectoryName string) {
if len(caseCorrectDirectoryName) == 0 {
fs.Stats.Error()

73
dropbox/nametree_test.go
View File

@ -5,32 +5,47 @@ import (
"github.com/ncw/rclone/fs"
dropboxapi "github.com/stacktic/dropbox"
"github.com/stretchr/testify/assert"
)
func assert(t *testing.T, shouldBeTrue bool, failMessage string) {
if !shouldBeTrue {
t.Fatal(failMessage)
}
}
func TestPutCaseCorrectDirectoryName(t *testing.T) {
errors := fs.Stats.GetErrors()
tree := newNameTree()
tree.PutCaseCorrectDirectoryName("a/b", "C")
assert(t, tree.CaseCorrectName == "", "Root CaseCorrectName should be empty")
assert.Equal(t, "", tree.CaseCorrectName, "Root CaseCorrectName should be empty")
a := tree.Directories["a"]
assert(t, a.CaseCorrectName == "", "CaseCorrectName at 'a' should be empty")
assert.Equal(t, "", a.CaseCorrectName, "CaseCorrectName at 'a' should be empty")
b := a.Directories["b"]
assert(t, b.CaseCorrectName == "", "CaseCorrectName at 'a/b' should be empty")
assert.Equal(t, "", b.CaseCorrectName, "CaseCorrectName at 'a/b' should be empty")
c := b.Directories["c"]
assert(t, c.CaseCorrectName == "C", "CaseCorrectName at 'a/b/c' should be 'C'")
assert.Equal(t, "C", c.CaseCorrectName, "CaseCorrectName at 'a/b/c' should be 'C'")
assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
assert.Equal(t, errors, fs.Stats.GetErrors(), "No errors should be reported")
}
func TestPutCaseCorrectPath(t *testing.T) {
errors := fs.Stats.GetErrors()
tree := newNameTree()
tree.PutCaseCorrectPath("A/b/C")
assert.Equal(t, "", tree.CaseCorrectName, "Root CaseCorrectName should be empty")
a := tree.Directories["a"]
assert.Equal(t, "A", a.CaseCorrectName, "CaseCorrectName at 'a' should be 'A'")
b := a.Directories["b"]
assert.Equal(t, "b", b.CaseCorrectName, "CaseCorrectName at 'a/b' should be 'b'")
c := b.Directories["c"]
assert.Equal(t, "C", c.CaseCorrectName, "CaseCorrectName at 'a/b/c' should be 'C'")
assert.Equal(t, errors, fs.Stats.GetErrors(), "No errors should be reported")
}
func TestPutCaseCorrectDirectoryNameEmptyComponent(t *testing.T) {
@ -41,7 +56,7 @@ func TestPutCaseCorrectDirectoryNameEmptyComponent(t *testing.T) {
tree.PutCaseCorrectDirectoryName("b/", "C")
tree.PutCaseCorrectDirectoryName("a//b", "C")
assert(t, fs.Stats.GetErrors() == errors+3, "3 errors should be reported")
assert.True(t, fs.Stats.GetErrors() == errors+3, "3 errors should be reported")
}
func TestPutCaseCorrectDirectoryNameEmptyParent(t *testing.T) {
@ -51,9 +66,9 @@ func TestPutCaseCorrectDirectoryNameEmptyParent(t *testing.T) {
tree.PutCaseCorrectDirectoryName("", "C")
c := tree.Directories["c"]
assert(t, c.CaseCorrectName == "C", "CaseCorrectName at 'c' should be 'C'")
assert.True(t, c.CaseCorrectName == "C", "CaseCorrectName at 'c' should be 'C'")
assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
}
func TestGetPathWithCorrectCase(t *testing.T) {
@ -61,12 +76,12 @@ func TestGetPathWithCorrectCase(t *testing.T) {
tree := newNameTree()
tree.PutCaseCorrectDirectoryName("a", "C")
assert(t, tree.GetPathWithCorrectCase("a/c") == nil, "Path for 'a' should not be available")
assert.True(t, tree.GetPathWithCorrectCase("a/c") == nil, "Path for 'a' should not be available")
tree.PutCaseCorrectDirectoryName("", "A")
assert(t, *tree.GetPathWithCorrectCase("a/c") == "/A/C", "Path for 'a/c' should be '/A/C'")
assert.True(t, *tree.GetPathWithCorrectCase("a/c") == "/A/C", "Path for 'a/c' should be '/A/C'")
assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
}
func TestPutAndWalk(t *testing.T) {
@ -78,15 +93,15 @@ func TestPutAndWalk(t *testing.T) {
numCalled := 0
walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) error {
assert(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath)
assert(t, entry.Path == "xxx", "entry.Path should be xxx")
assert.True(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath)
assert.True(t, entry.Path == "xxx", "entry.Path should be xxx")
numCalled++
return nil
}
err := tree.WalkFiles("", walkFunc)
assert(t, err == nil, "No error should be returned")
assert(t, numCalled == 1, "walk func should be called only once")
assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
assert.True(t, err == nil, "No error should be returned")
assert.True(t, numCalled == 1, "walk func should be called only once")
assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
}
func TestPutAndWalkWithPrefix(t *testing.T) {
@ -98,15 +113,15 @@ func TestPutAndWalkWithPrefix(t *testing.T) {
numCalled := 0
walkFunc := func(caseCorrectFilePath string, entry *dropboxapi.Entry) error {
assert(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath)
assert(t, entry.Path == "xxx", "entry.Path should be xxx")
assert.True(t, caseCorrectFilePath == "A/F", "caseCorrectFilePath should be A/F, not "+caseCorrectFilePath)
assert.True(t, entry.Path == "xxx", "entry.Path should be xxx")
numCalled++
return nil
}
err := tree.WalkFiles("A", walkFunc)
assert(t, err == nil, "No error should be returned")
assert(t, numCalled == 1, "walk func should be called only once")
assert(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
assert.True(t, err == nil, "No error should be returned")
assert.True(t, numCalled == 1, "walk func should be called only once")
assert.True(t, fs.Stats.GetErrors() == errors, "No errors should be reported")
}
func TestPutAndWalkIncompleteTree(t *testing.T) {
@ -120,6 +135,6 @@ func TestPutAndWalkIncompleteTree(t *testing.T) {
return nil
}
err := tree.WalkFiles("", walkFunc)
assert(t, err == nil, "No error should be returned")
assert(t, fs.Stats.GetErrors() == errors+1, "One error should be reported")
assert.True(t, err == nil, "No error should be returned")
assert.True(t, fs.Stats.GetErrors() == errors+1, "One error should be reported")
}

15
fs/fs.go
View File

@ -40,6 +40,7 @@ var (
ErrorDirNotFound = fmt.Errorf("Directory not found")
ErrorLevelNotSupported = fmt.Errorf("Level value not supported")
ErrorListAborted = fmt.Errorf("List aborted")
ErrorListOnlyRoot = fmt.Errorf("Can only list from root")
)
// RegInfo provides information about a filesystem
@ -98,10 +99,14 @@ func Register(info *RegInfo) {
type Fs interface {
Info
// List the objects and directories of the Fs
// List the objects and directories of the Fs starting from dir
//
// This should return ErrDirNotFound if the directory isn't found.
List(ListOpts)
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound (using out.SetError())
// if the directory isn't found.
List(out ListOpts, dir string)
// NewFsObject finds the Object at remote. Returns nil if can't be found
NewFsObject(remote string) Object
@ -312,10 +317,10 @@ func NewLister() *Lister {
// Start starts a go routine listing the Fs passed in. It returns the
// same Lister that was passed in for convenience.
func (o *Lister) Start(f Fs) *Lister {
func (o *Lister) Start(f Fs, dir string) *Lister {
o.results = make(chan listerResult, o.buffer)
go func() {
f.List(o)
f.List(o, dir)
}()
return o
}

6
fs/limited.go
View File

@ -38,8 +38,12 @@ func (f *Limited) String() string {
}
// List the Fs into a channel
func (f *Limited) List(opts ListOpts) {
func (f *Limited) List(opts ListOpts, dir string) {
defer opts.Finished()
if dir != "" {
opts.SetError(ErrorListOnlyRoot)
return
}
for _, obj := range f.objects {
if opts.Add(obj) {
return

32
fs/operations.go
View File

@ -454,16 +454,17 @@ func DeleteFiles(toBeDeleted ObjectsChan) {
}
// Read a map of Object.Remote to Object for the given Fs.
// dir is the start directory, "" for root
// If includeAll is specified all files will be added,
// otherwise only files passing the filter will be added.
func readFilesMap(fs Fs, includeAll bool) (files map[string]Object, err error) {
func readFilesMap(fs Fs, includeAll bool, dir string) (files map[string]Object, err error) {
files = make(map[string]Object)
normalised := make(map[string]struct{})
list := NewLister()
if !includeAll {
list.SetFilter(Config.Filter)
}
list.Start(fs)
list.Start(fs, dir)
for {
o, err := list.GetObject()
if err != nil {
@ -494,14 +495,15 @@ func readFilesMap(fs Fs, includeAll bool) (files map[string]Object, err error) {
}
// readFilesMaps runs readFilesMap on fdst and fsrc at the same time
func readFilesMaps(fdst Fs, fdstIncludeAll bool, fsrc Fs, fsrcIncludeAll bool) (dstFiles, srcFiles map[string]Object, err error) {
// dir is the start directory, "" for root
func readFilesMaps(fdst Fs, fdstIncludeAll bool, fsrc Fs, fsrcIncludeAll bool, dir string) (dstFiles, srcFiles map[string]Object, err error) {
var wg sync.WaitGroup
var srcErr, dstErr error
list := func(fs Fs, includeAll bool, pMap *map[string]Object, pErr *error) {
defer wg.Done()
Log(fs, "Building file list")
dstFiles, listErr := readFilesMap(fs, includeAll)
dstFiles, listErr := readFilesMap(fs, includeAll, dir)
if listErr != nil {
ErrorLog(fs, "Error building file list: %v", listErr)
*pErr = listErr
@ -535,7 +537,9 @@ func Same(fdst, fsrc Fs) bool {
// If Delete is true then it deletes any files in fdst that aren't in fsrc
//
// If DoMove is true then files will be moved instead of copied
func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
//
// dir is the start directory, "" for root
func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool, dir string) error {
if Same(fdst, fsrc) {
ErrorLog(fdst, "Nothing to do as source and destination are the same")
return nil
@ -547,7 +551,7 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
}
// Read the files of both source and destination in parallel
dstFiles, srcFiles, err := readFilesMaps(fdst, Config.Filter.DeleteExcluded, fsrc, false)
dstFiles, srcFiles, err := readFilesMaps(fdst, Config.Filter.DeleteExcluded, fsrc, false, dir)
if err != nil {
return err
}
@ -651,12 +655,12 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
// Sync fsrc into fdst
func Sync(fdst, fsrc Fs) error {
return syncCopyMove(fdst, fsrc, true, false)
return syncCopyMove(fdst, fsrc, true, false, "")
}
// CopyDir copies fsrc into fdst
func CopyDir(fdst, fsrc Fs) error {
return syncCopyMove(fdst, fsrc, false, false)
return syncCopyMove(fdst, fsrc, false, false, "")
}
// MoveDir moves fsrc into fdst
@ -684,7 +688,7 @@ func MoveDir(fdst, fsrc Fs) error {
}
// Now move the files
err := syncCopyMove(fdst, fsrc, false, true)
err := syncCopyMove(fdst, fsrc, false, true, "")
if err != nil || Stats.Errored() {
ErrorLog(fdst, "Not deleting files as there were IO errors")
return err
@ -732,7 +736,7 @@ func checkIdentical(dst, src Object) bool {
// Check the files in fsrc and fdst according to Size and hash
func Check(fdst, fsrc Fs) error {
dstFiles, srcFiles, err := readFilesMaps(fdst, false, fsrc, false)
dstFiles, srcFiles, err := readFilesMaps(fdst, false, fsrc, false, "")
if err != nil {
return err
}
@ -800,7 +804,7 @@ func Check(fdst, fsrc Fs) error {
//
// Lists in parallel which may get them out of order
func ListFn(f Fs, fn func(Object)) error {
list := NewLister().SetFilter(Config.Filter).Start(f)
list := NewLister().SetFilter(Config.Filter).Start(f, "")
var wg sync.WaitGroup
wg.Add(Config.Checkers)
for i := 0; i < Config.Checkers; i++ {
@ -909,7 +913,7 @@ func Count(f Fs) (objects int64, size int64, err error) {
// ListDir lists the directories/buckets/containers in the Fs to the supplied writer
func ListDir(f Fs, w io.Writer) error {
list := NewLister().SetLevel(1).Start(f)
list := NewLister().SetLevel(1).Start(f, "")
for {
dir, err := list.GetDir()
if err != nil {
@ -976,7 +980,7 @@ func Purge(f Fs) error {
}
if doFallbackPurge {
// DeleteFiles and Rmdir observe --dry-run
list := NewLister().Start(f)
list := NewLister().Start(f, "")
DeleteFiles(listToChan(list))
err = Rmdir(f)
}
@ -1132,7 +1136,7 @@ func (mode DeduplicateMode) String() string {
func Deduplicate(f Fs, mode DeduplicateMode) error {
Log(f, "Looking for duplicates using %v mode.", mode)
files := map[string][]Object{}
list := NewLister().Start(f)
list := NewLister().Start(f, "")
for {
o, err := list.GetObject()
if err != nil {

4
fs/operations_test.go
View File

@ -140,7 +140,7 @@ func NewRun(t *testing.T) *Run {
r = new(Run)
*r = *oneRun
r.cleanRemote = func() {
list := fs.NewLister().Start(r.fremote)
list := fs.NewLister().Start(r.fremote, "")
for {
o, err := list.GetObject()
if err != nil {
@ -1181,7 +1181,7 @@ func TestDeduplicateRename(t *testing.T) {
t.Fatalf("fs.Deduplicate returned error: %v", err)
}
list := fs.NewLister().Start(r.fremote)
list := fs.NewLister().Start(r.fremote, "")
for {
o, err := list.GetObject()
if err != nil {

2
fstest/fstest.go
View File

@ -158,7 +158,7 @@ func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision ti
const retries = 6
sleep := time.Second / 2
for i := 1; i <= retries; i++ {
objs, err = fs.NewLister().Start(f).GetObjects()
objs, err = fs.NewLister().Start(f, "").GetObjects()
if err != nil && err != fs.ErrorDirNotFound {
t.Fatalf("Error listing: %v", err)
}

67
fstest/fstests/fstests.go
View File

@ -11,12 +11,15 @@ import (
"io"
"log"
"os"
"path"
"strings"
"testing"
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var (
@ -129,10 +132,8 @@ func TestFsListEmpty(t *testing.T) {
// TestFsListDirEmpty tests listing the directories from an empty directory
func TestFsListDirEmpty(t *testing.T) {
skipIfNotOk(t)
dirs, err := fs.NewLister().SetLevel(1).Start(remote).GetDirs()
if err != nil {
t.Fatal(err)
}
dirs, err := fs.NewLister().SetLevel(1).Start(remote, "").GetDirs()
require.NoError(t, err)
for _, dir := range dirs {
t.Errorf("Found unexpected item %q", dir.Name)
}
@ -197,10 +198,8 @@ func TestFsListDirFile2(t *testing.T) {
skipIfNotOk(t)
found := false
for i := 1; i <= eventualConsistencyRetries; i++ {
dirs, err := fs.NewLister().SetLevel(1).Start(remote).GetDirs()
if err != nil {
t.Fatal(err)
}
dirs, err := fs.NewLister().SetLevel(1).Start(remote, "").GetDirs()
require.NoError(t, err)
for _, dir := range dirs {
if dir.Name != `hello? sausage` && dir.Name != `hello_ sausage` {
t.Errorf("Found unexpected item %q", dir.Name)
@ -227,10 +226,8 @@ func TestFsListDirRoot(t *testing.T) {
t.Fatalf("Failed to make remote %q: %v", RemoteName, err)
}
found := false
dirs, err := fs.NewLister().SetLevel(1).Start(rootRemote).GetDirs()
if err != nil {
t.Fatal(err)
}
dirs, err := fs.NewLister().SetLevel(1).Start(rootRemote, "").GetDirs()
require.NoError(t, err)
for _, dir := range dirs {
if dir.Name == subRemoteLeaf {
found = true
@ -241,42 +238,22 @@ func TestFsListDirRoot(t *testing.T) {
}
}
// TestFsListRoot tests List works in the root
func TestFsListRoot(t *testing.T) {
// TestFsListSubdir tests List works for a subdirectory
func TestFsListSubdir(t *testing.T) {
skipIfNotOk(t)
rootRemote, err := fs.NewFs(RemoteName)
if err != nil {
t.Fatalf("Failed to make remote %q: %v", RemoteName, err)
}
// Should either find file1 and file2 or nothing
found1 := false
f1 := subRemoteLeaf + "/" + file1.Path
found2 := false
f2 := subRemoteLeaf + "/" + file2.Path
f2Alt := subRemoteLeaf + "/" + file2.WinPath
count := 0
objs, err := fs.NewLister().Start(rootRemote).GetObjects()
if err != nil {
t.Fatal(err)
}
for _, obj := range objs {
count++
if obj.Remote() == f1 {
found1 = true
}
if obj.Remote() == f2 || obj.Remote() == f2Alt {
found2 = true
test := func(fileName string) bool {
dir, _ := path.Split(fileName)
dir = dir[:len(dir)-1]
objs, err := fs.NewLister().Start(remote, dir).GetObjects()
if err == fs.ErrorDirNotFound {
return false
}
require.NoError(t, err)
require.Len(t, objs, 1)
assert.Equal(t, fileName, objs[0].Remote())
return true
}
if count == 0 {
// Nothing found is OK
return
}
if found1 && found2 {
// Both found is OK
return
}
t.Errorf("Didn't find %q (%v) and %q (%v) or no files (count %d)", f1, found1, f2, found2, count)
assert.True(t, test(file2.Path) || test(file2.WinPath), "normal and alternative lists failed")
}
// TestFsListFile1 tests file present

30
googlecloudstorage/googlecloudstorage.go
View File

@ -300,9 +300,15 @@ type listFn func(remote string, object *storage.Object, isDirectory bool) error
// list the objects into the function supplied
//
// dir is the starting directory, "" for root
//
// If directories is set it only sends directories
func (f *Fs) list(level int, fn listFn) error {
list := f.svc.Objects.List(f.bucket).Prefix(f.root).MaxResults(listChunks)
func (f *Fs) list(dir string, level int, fn listFn) error {
root := f.root
if dir != "" {
root += dir + "/"
}
list := f.svc.Objects.List(f.bucket).Prefix(root).MaxResults(listChunks)
switch level {
case 1:
list = list.Delimiter("/")
@ -310,7 +316,7 @@ func (f *Fs) list(level int, fn listFn) error {
default:
return fs.ErrorLevelNotSupported
}
rootLength := len(f.root)
rootLength := len(root)
for {
objects, err := list.Do()
if err != nil {
@ -329,7 +335,7 @@ func (f *Fs) list(level int, fn listFn) error {
}
}
for _, object := range objects.Items {
if !strings.HasPrefix(object.Name, f.root) {
if !strings.HasPrefix(object.Name, root) {
fs.Log(f, "Odd name received %q", object.Name)
continue
}
@ -348,14 +354,14 @@ func (f *Fs) list(level int, fn listFn) error {
}
// listFiles lists files and directories to out
func (f *Fs) listFiles(out fs.ListOpts) {
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
if f.bucket == "" {
out.SetError(fmt.Errorf("Can't list objects at root - choose a bucket using lsd"))
return
}
// List the objects
err := f.list(out.Level(), func(remote string, object *storage.Object, isDirectory bool) error {
err := f.list(dir, out.Level(), func(remote string, object *storage.Object, isDirectory bool) error {
if isDirectory {
dir := &fs.Dir{
Name: remote,
@ -385,8 +391,12 @@ func (f *Fs) listFiles(out fs.ListOpts) {
}
// listBuckets lists the buckets to out
func (f *Fs) listBuckets(out fs.ListOpts) {
func (f *Fs) listBuckets(out fs.ListOpts, dir string) {
defer out.Finished()
if dir != "" {
out.SetError(fs.ErrorListOnlyRoot)
return
}
if f.projectNumber == "" {
out.SetError(errors.New("Can't list buckets without project number"))
return
@ -416,11 +426,11 @@ func (f *Fs) listBuckets(out fs.ListOpts) {
}
// List lists the path to out
func (f *Fs) List(out fs.ListOpts) {
func (f *Fs) List(out fs.ListOpts, dir string) {
if f.bucket == "" {
f.listBuckets(out)
f.listBuckets(out, dir)
} else {
f.listFiles(out)
f.listFiles(out, dir)
}
return
}

2
googlecloudstorage/googlecloudstorage_test.go
View File

@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

2
hubic/hubic_test.go
View File

@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

54
local/local.go
View File

@ -210,13 +210,12 @@ func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (su
// List the path into out
//
// Ignores everything which isn't Storable, eg links etc
func (f *Fs) List(out fs.ListOpts) {
func (f *Fs) List(out fs.ListOpts, dir string) {
defer out.Finished()
_, err := os.Stat(f.root)
root := path.Join(f.root, dir)
_, err := os.Stat(root)
if err != nil {
out.SetError(fs.ErrorDirNotFound)
fs.Stats.Error()
fs.ErrorLog(f, "Directory not found: %s: %s", f.root, err)
return
}
@ -226,7 +225,7 @@ func (f *Fs) List(out fs.ListOpts) {
// Start the process
traversing.Add(1)
in <- listArgs{remote: "", dirpath: f.root, level: out.Level() - 1}
in <- listArgs{remote: "", dirpath: root, level: out.Level() - 1}
for i := 0; i < fs.Config.Checkers; i++ {
wg.Add(1)
go func() {
@ -276,51 +275,6 @@ func (f *Fs) cleanUtf8(name string) string {
return name
}
/*
// ListDir walks the path returning a channel of FsObjects
func (f *Fs) ListDir(out fs.ListDirOpts) {
defer out.Finished()
items, err := ioutil.ReadDir(f.root)
if err != nil {
fs.Stats.Error()
fs.ErrorLog(f, "Couldn't find read directory: %s", err)
out.SetError(err)
return
}
for _, item := range items {
if item.IsDir() {
dir := &fs.Dir{
Name: f.cleanUtf8(item.Name()),
When: item.ModTime(),
Bytes: 0,
Count: 0,
}
// Go down the tree to count the files and directories
dirpath := f.filterPath(filepath.Join(f.root, item.Name()))
err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error {
if err != nil {
fs.Stats.Error()
fs.ErrorLog(f, "Failed to open directory: %s: %s", path, err)
out.SetError(err)
} else {
dir.Count++
dir.Bytes += fi.Size()
}
return nil
})
if err != nil {
out.SetError(err)
fs.Stats.Error()
fs.ErrorLog(f, "Failed to open directory: %s: %s", dirpath, err)
}
if out.Add(dir) {
return
}
}
}
}
*/
// Put the FsObject to the local filesystem
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
remote := src.Remote()

2
local/local_test.go
View File

@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

4
onedrive/onedrive.go
View File

@ -405,8 +405,8 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.
}
// List walks the path returning files and directories into out
func (f *Fs) List(out fs.ListOpts) {
f.dirCache.List(f, out)
func (f *Fs) List(out fs.ListOpts, dir string) {
f.dirCache.List(f, out, dir)
}
// Creates from the parameters passed in a half finished Object which

2
onedrive/onedrive_test.go
View File

@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

26
s3/s3.go
View File

@ -369,8 +369,14 @@ type listFn func(remote string, object *s3.Object, isDirectory bool) error
// list the objects into the function supplied
//
// dir is the starting directory, "" for root
//
// Level is the level of the recursion
func (f *Fs) list(level int, fn listFn) error {
func (f *Fs) list(dir string, level int, fn listFn) error {
root := f.root
if dir != "" {
root += dir + "/"
}
maxKeys := int64(listChunkSize)
delimiter := ""
switch level {
@ -386,7 +392,7 @@ func (f *Fs) list(level int, fn listFn) error {
req := s3.ListObjectsInput{
Bucket: &f.bucket,
Delimiter: &delimiter,
Prefix: &f.root,
Prefix: &root,
MaxKeys: &maxKeys,
Marker: marker,
}
@ -442,7 +448,7 @@ func (f *Fs) list(level int, fn listFn) error {
}
// listFiles lists files and directories to out
func (f *Fs) listFiles(out fs.ListOpts) {
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
if f.bucket == "" {
// Return no objects at top level list
@ -450,7 +456,7 @@ func (f *Fs) listFiles(out fs.ListOpts) {
return
}
// List the objects and directories
err := f.list(out.Level(), func(remote string, object *s3.Object, isDirectory bool) error {
err := f.list(dir, out.Level(), func(remote string, object *s3.Object, isDirectory bool) error {
if isDirectory {
size := int64(0)
if object.Size != nil {
@ -484,8 +490,12 @@ func (f *Fs) listFiles(out fs.ListOpts) {
}
// listBuckets lists the buckets to out
func (f *Fs) listBuckets(out fs.ListOpts) {
func (f *Fs) listBuckets(out fs.ListOpts, dir string) {
defer out.Finished()
if dir != "" {
out.SetError(fs.ErrorListOnlyRoot)
return
}
req := s3.ListBucketsInput{}
resp, err := f.c.ListBuckets(&req)
if err != nil {
@ -506,11 +516,11 @@ func (f *Fs) listBuckets(out fs.ListOpts) {
}
// List lists files and directories to out
func (f *Fs) List(out fs.ListOpts) {
func (f *Fs) List(out fs.ListOpts, dir string) {
if f.bucket == "" {
f.listBuckets(out)
f.listBuckets(out, dir)
} else {
f.listFiles(out)
f.listFiles(out, dir)
}
return
}

2
s3/s3_test.go
View File

@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

37
swift/swift.go
View File

@ -260,10 +260,14 @@ type listFn func(remote string, object *swift.Object, isDirectory bool) error
// the container and root supplied
//
// Level is the level of the recursion
func (f *Fs) listContainerRoot(container, root string, level int, fn listFn) error {
func (f *Fs) listContainerRoot(container, root string, dir string, level int, fn listFn) error {
prefix := root
if dir != "" {
prefix += dir + "/"
}
// Options for ObjectsWalk
opts := swift.ObjectsOpts{
Prefix: root,
Prefix: prefix,
Limit: 256,
}
switch level {
@ -302,21 +306,19 @@ func (f *Fs) listContainerRoot(container, root string, level int, fn listFn) err
}
// list the objects into the function supplied
func (f *Fs) list(level int, fn listFn) error {
return f.listContainerRoot(f.container, f.root, level, fn)
func (f *Fs) list(dir string, level int, fn listFn) error {
return f.listContainerRoot(f.container, f.root, dir, level, fn)
}
// listFiles walks the path returning a channel of FsObjects
//
// if ignoreStorable is set then it outputs the file even if Storable() is false
func (f *Fs) listFiles(out fs.ListOpts, ignoreStorable bool) {
func (f *Fs) listFiles(out fs.ListOpts, dir string) {
defer out.Finished()
if f.container == "" {
out.SetError(errors.New("Can't list objects at root - choose a container using lsd"))
return
}
// List the objects
err := f.list(out.Level(), func(remote string, object *swift.Object, isDirectory bool) error {
err := f.list(dir, out.Level(), func(remote string, object *swift.Object, isDirectory bool) error {
if isDirectory {
dir := &fs.Dir{
Name: remote,
@ -329,8 +331,7 @@ func (f *Fs) listFiles(out fs.ListOpts, ignoreStorable bool) {
} else {
if o := f.newFsObjectWithInfo(remote, object); o != nil {
// Storable does a full metadata read on 0 size objects which might be dynamic large objects
storable := o.Storable()
if storable || ignoreStorable {
if o.Storable() {
if out.Add(o) {
return fs.ErrorListAborted
}
@ -348,8 +349,12 @@ func (f *Fs) listFiles(out fs.ListOpts, ignoreStorable bool) {
}
// listContainers lists the containers
func (f *Fs) listContainers(out fs.ListOpts) {
func (f *Fs) listContainers(out fs.ListOpts, dir string) {
defer out.Finished()
if dir != "" {
out.SetError(fs.ErrorListOnlyRoot)
return
}
containers, err := f.c.ContainersAll(nil)
if err != nil {
out.SetError(err)
@ -368,11 +373,11 @@ func (f *Fs) listContainers(out fs.ListOpts) {
}
// List walks the path returning files and directories to out
func (f *Fs) List(out fs.ListOpts) {
func (f *Fs) List(out fs.ListOpts, dir string) {
if f.container == "" {
f.listContainers(out)
f.listContainers(out, dir)
} else {
f.listFiles(out, false)
f.listFiles(out, dir)
}
return
}
@ -428,7 +433,7 @@ func (f *Fs) Purge() error {
toBeDeleted := make(chan fs.Object, fs.Config.Transfers)
var err error
go func() {
err = f.list(fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error {
err = f.list("", fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error {
if !isDirectory {
if o := f.newFsObjectWithInfo(remote, object); o != nil {
toBeDeleted <- o
@ -625,7 +630,7 @@ func min(x, y int64) int64 {
// if except is passed in then segments with that prefix won't be deleted
func (o *Object) removeSegments(except string) error {
segmentsRoot := o.fs.root + o.remote + "/"
err := o.fs.listContainerRoot(o.fs.segmentsContainer, segmentsRoot, fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error {
err := o.fs.listContainerRoot(o.fs.segmentsContainer, segmentsRoot, "", fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error {
if isDirectory {
return nil
}

2
swift/swift_test.go
View File

@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }

18
yandex/yandex.go
View File

@ -200,7 +200,7 @@ func (f *Fs) listDir(fn listFn) (err error) {
// list the objects into the function supplied
//
// This does a flat listing of all the files in the drive
func (f *Fs) list(fn listFn) error {
func (f *Fs) list(dir string, fn listFn) error {
//request files list. list is divided into pages. We send request for each page
//items per page is limited by limit
//TODO may be add config parameter for the items per page limit
@ -211,6 +211,10 @@ func (f *Fs) list(fn listFn) error {
var opt yandex.FlatFileListRequestOptions
opt.Limit = &limit
opt.Offset = &offset
prefix := f.diskRoot
if dir != "" {
prefix += dir + "/"
}
//query each page of list until itemCount is less then limit
for {
//send request
@ -223,7 +227,7 @@ func (f *Fs) list(fn listFn) error {
//list files
for _, item := range info.Items {
// filter file list and get only files we need
if strings.HasPrefix(item.Path, f.diskRoot) {
if strings.HasPrefix(item.Path, prefix) {
//trim root folder from filename
var name = strings.TrimPrefix(item.Path, f.diskRoot)
err = fn(name, &item, false)
@ -244,7 +248,7 @@ func (f *Fs) list(fn listFn) error {
}
// List walks the path returning a channel of FsObjects
func (f *Fs) List(out fs.ListOpts) {
func (f *Fs) List(out fs.ListOpts, dir string) {
defer out.Finished()
listItem := func(remote string, object *yandex.ResourceInfoResponse, isDirectory bool) error {
@ -275,9 +279,13 @@ func (f *Fs) List(out fs.ListOpts) {
var err error
switch out.Level() {
case 1:
err = f.listDir(listItem)
if dir == "" {
err = f.listDir(listItem)
} else {
err = f.list(dir, listItem)
}
case fs.MaxLevel:
err = f.list(listItem)
err = f.list(dir, listItem)
default:
out.SetError(fs.ErrorLevelNotSupported)
}

2
yandex/yandex_test.go
View File

@ -30,7 +30,7 @@ func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }