From 8a6a8b9623bea1f600ad15eec2dc74ed37e65110 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 11 Jun 2017 22:43:31 +0100 Subject: [PATCH] Change List interface and add ListR optional interface This simplifies the implementation of remotes. The only required interface is now `List` which is a simple one level directory list. Optionally remotes may implement `ListR` if they have an efficient way of doing a recursive list. --- amazonclouddrive/amazonclouddrive.go | 70 ++-- amazonclouddrive/amazonclouddrive_test.go | 5 + b2/b2.go | 151 ++++--- b2/b2_test.go | 5 + crypt/crypt.go | 169 ++++---- crypt/crypt2_test.go | 5 + crypt/crypt3_test.go | 5 + crypt/crypt_test.go | 5 + dircache/list.go | 82 ---- drive/drive.go | 90 ++-- drive/drive_test.go | 5 + dropbox/dropbox.go | 62 ++- dropbox/dropbox_test.go | 5 + fs/fs.go | 87 ++-- fs/lister.go | 314 -------------- fs/lister_test.go | 384 ------------------ fs/operations.go | 58 +-- fs/walk.go | 109 ++--- fs/walk_test.go | 28 +- fstest/fstests/fstests.go | 45 ++ ftp/ftp.go | 57 +-- ftp/ftp_test.go | 5 + googlecloudstorage/googlecloudstorage.go | 145 ++++--- googlecloudstorage/googlecloudstorage_test.go | 5 + hubic/hubic_test.go | 5 + local/local.go | 118 ++---- local/local_test.go | 5 + onedrive/onedrive.go | 75 ++-- onedrive/onedrive_test.go | 5 + s3/s3.go | 149 ++++--- s3/s3_test.go | 5 + sftp/sftp.go | 81 ++-- sftp/sftp_test.go | 5 + swift/swift.go | 126 +++--- swift/swift_test.go | 5 + yandex/yandex.go | 150 ++++--- yandex/yandex_test.go | 5 + 37 files changed, 994 insertions(+), 1636 deletions(-) delete mode 100644 dircache/list.go delete mode 100644 fs/lister.go delete mode 100644 fs/lister_test.go diff --git a/amazonclouddrive/amazonclouddrive.go b/amazonclouddrive/amazonclouddrive.go index 8655bbf10..bf4f42a71 100644 --- a/amazonclouddrive/amazonclouddrive.go +++ b/amazonclouddrive/amazonclouddrive.go @@ -3,7 +3,6 @@ package amazonclouddrive /* - FIXME make searching for directory in id and file in id more efficient - use the name: search parameter - remember the escaping rules - use Folder GetNode and GetFile @@ -399,47 +398,58 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly return } -// ListDir reads the directory specified by the job into out, returning any more jobs -func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.ListDirJob, err error) { - fs.Debugf(f, "Reading %q", job.Path) +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + err = f.dirCache.FindRoot(false) + if err != nil { + return nil, err + } + directoryID, err := f.dirCache.FindDir(dir, false) + if err != nil { + return nil, err + } maxTries := fs.Config.LowLevelRetries + var iErr error for tries := 1; tries <= maxTries; tries++ { - _, err = f.listAll(job.DirID, "", false, false, func(node *acd.Node) bool { - remote := job.Path + *node.Name + entries = nil + _, err = f.listAll(directoryID, "", false, false, func(node *acd.Node) bool { + remote := path.Join(dir, *node.Name) switch *node.Kind { case folderKind: - if out.IncludeDirectory(remote) { - // cache the directory ID for later lookups - f.dirCache.Put(remote, *node.Id) - dir := &fs.Dir{ - Name: remote, - Bytes: -1, - Count: -1, - } - dir.When, _ = time.Parse(timeFormat, *node.ModifiedDate) // FIXME - if out.AddDir(dir) { - return true - } - if job.Depth > 0 { - jobs = append(jobs, dircache.ListDirJob{DirID: *node.Id, Path: remote + "/", Depth: job.Depth - 1}) - } + // cache the directory ID for later lookups + f.dirCache.Put(remote, *node.Id) + d := &fs.Dir{ + Name: remote, + Bytes: -1, + Count: -1, } + d.When, _ = time.Parse(timeFormat, *node.ModifiedDate) // FIXME + entries = append(entries, d) case fileKind: o, err := f.newObjectWithInfo(remote, node) if err != nil { - out.SetError(err) - return true - } - if out.Add(o) { + iErr = err return true } + entries = append(entries, o) default: // ignore ASSET etc } return false }) + if iErr != nil { + return nil, iErr + } if fs.IsRetryError(err) { - fs.Debugf(f, "Directory listing error for %q: %v - low level retry %d/%d", job.Path, err, tries, maxTries) + fs.Debugf(f, "Directory listing error for %q: %v - low level retry %d/%d", dir, err, tries, maxTries) continue } if err != nil { @@ -447,13 +457,7 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache. } break } - fs.Debugf(f, "Finished reading %q", job.Path) - return jobs, err -} - -// List walks the path returning iles and directories into out -func (f *Fs) List(out fs.ListOpts, dir string) { - f.dirCache.List(f, out, dir) + return entries, nil } // checkUpload checks to see if an error occurred after the file was diff --git a/amazonclouddrive/amazonclouddrive_test.go b/amazonclouddrive/amazonclouddrive_test.go index c92e81cb6..20dad31c1 100644 --- a/amazonclouddrive/amazonclouddrive_test.go +++ b/amazonclouddrive/amazonclouddrive_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/b2/b2.go b/b2/b2.go index b1b57ef02..739b807ed 100644 --- a/b2/b2.go +++ b/b2/b2.go @@ -438,18 +438,14 @@ var errEndList = errors.New("end list") // than 1000) // // If hidden is set then it will list the hidden (deleted) files too. -func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool, fn listFn) error { +func (f *Fs) list(dir string, recurse bool, prefix string, limit int, hidden bool, fn listFn) error { root := f.root if dir != "" { root += dir + "/" } delimiter := "" - switch level { - case 1: + if !recurse { delimiter = "/" - case fs.MaxLevel: - default: - return fs.ErrorLevelNotSupported } bucketID, err := f.getBucketID() if err != nil { @@ -497,7 +493,7 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool, } remote := file.Name[len(f.root):] // Check for directory - isDirectory := level != 0 && strings.HasSuffix(remote, "/") + isDirectory := strings.HasSuffix(remote, "/") if isDirectory { remote = remote[:len(remote)-1] } @@ -522,83 +518,120 @@ func (f *Fs) list(dir string, level int, prefix string, limit int, hidden bool, return nil } -// listFiles walks the path returning files and directories to out -func (f *Fs) listFiles(out fs.ListOpts, dir string) { - defer out.Finished() - // List the objects +// Convert a list item into a BasicInfo +func (f *Fs) itemToDirEntry(remote string, object *api.File, isDirectory bool, last *string) (fs.BasicInfo, error) { + if isDirectory { + d := &fs.Dir{ + Name: remote, + Bytes: -1, + Count: -1, + } + return d, nil + } + if remote == *last { + remote = object.UploadTimestamp.AddVersion(remote) + } else { + *last = remote + } + // hide objects represent deleted files which we don't list + if object.Action == "hide" { + return nil, nil + } + o, err := f.newObjectWithInfo(remote, object) + if err != nil { + return nil, err + } + return o, nil +} + +// listDir lists a single directory +func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { last := "" - err := f.list(dir, out.Level(), "", 0, *b2Versions, func(remote string, object *api.File, isDirectory bool) error { - if isDirectory { - dir := &fs.Dir{ - Name: remote, - Bytes: -1, - Count: -1, - } - if out.AddDir(dir) { - return fs.ErrorListAborted - } - } else { - if remote == last { - remote = object.UploadTimestamp.AddVersion(remote) - } else { - last = remote - } - // hide objects represent deleted files which we don't list - if object.Action == "hide" { - return nil - } - o, err := f.newObjectWithInfo(remote, object) - if err != nil { - return err - } - if out.Add(o) { - return fs.ErrorListAborted - } + err = f.list(dir, false, "", 0, *b2Versions, func(remote string, object *api.File, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory, &last) + if err != nil { + return err + } + if entry != nil { + entries = append(entries, entry) } return nil }) if err != nil { - out.SetError(err) + return nil, err } + return entries, nil } // listBuckets returns all the buckets to out -func (f *Fs) listBuckets(out fs.ListOpts, dir string) { - defer out.Finished() +func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) { if dir != "" { - out.SetError(fs.ErrorListOnlyRoot) - return + return nil, fs.ErrorListBucketRequired } - err := f.listBucketsToFn(func(bucket *api.Bucket) error { - dir := &fs.Dir{ + err = f.listBucketsToFn(func(bucket *api.Bucket) error { + d := &fs.Dir{ Name: bucket.Name, Bytes: -1, Count: -1, } - if out.AddDir(dir) { - return fs.ErrorListAborted - } + entries = append(entries, d) return nil }) if err != nil { - out.SetError(err) + return nil, err } + return entries, nil } -// List walks the path returning files and directories to out -func (f *Fs) List(out fs.ListOpts, dir string) { +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { if f.bucket == "" { - f.listBuckets(out, dir) - } else { - f.listFiles(out, dir) + return f.listBuckets(dir) } - return + return f.listDir(dir) } // ListR lists the objects and directories of the Fs starting // from dir recursively into out. -func (f *Fs) ListR(out fs.ListOpts, dir string) { - f.List(out, dir) // FIXME +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + if f.bucket == "" { + return fs.ErrorListBucketRequired + } + list := fs.NewListRHelper(callback) + last := "" + err = f.list(dir, true, "", 0, *b2Versions, func(remote string, object *api.File, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory, &last) + if err != nil { + return err + } + return list.Add(entry) + }) + if err != nil { + return err + } + return list.Flush() } // listBucketFn is called from listBucketsToFn to handle a bucket @@ -816,7 +849,7 @@ func (f *Fs) purge(oldOnly bool) error { }() } last := "" - checkErr(f.list("", fs.MaxLevel, "", 0, true, func(remote string, object *api.File, isDirectory bool) error { + checkErr(f.list("", true, "", 0, true, func(remote string, object *api.File, isDirectory bool) error { if !isDirectory { fs.Stats.Checking(remote) if oldOnly && last != remote { @@ -961,7 +994,7 @@ func (o *Object) readMetaData() (err error) { maxSearched = maxVersions } var info *api.File - err = o.fs.list("", fs.MaxLevel, baseRemote, maxSearched, *b2Versions, func(remote string, object *api.File, isDirectory bool) error { + err = o.fs.list("", true, baseRemote, maxSearched, *b2Versions, func(remote string, object *api.File, isDirectory bool) error { if isDirectory { return nil } diff --git a/b2/b2_test.go b/b2/b2_test.go index 1df0cea65..8817d8181 100644 --- a/b2/b2_test.go +++ b/b2/b2_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/crypt/crypt.go b/crypt/crypt.go index 57d8992e3..4f32ff4c5 100644 --- a/crypt/crypt.go +++ b/crypt/crypt.go @@ -6,7 +6,6 @@ import ( "io" "path" "strings" - "sync" "github.com/ncw/rclone/fs" "github.com/pkg/errors" @@ -143,9 +142,91 @@ func (f *Fs) String() string { return fmt.Sprintf("Encrypted drive '%s:%s'", f.name, f.root) } -// List the Fs into a channel -func (f *Fs) List(opts fs.ListOpts, dir string) { - f.Fs.List(f.newListOpts(opts, dir), f.cipher.EncryptDirName(dir)) +// Encrypt an object file name to entries. +func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) { + remote := obj.Remote() + decryptedRemote, err := f.cipher.DecryptFileName(remote) + if err != nil { + fs.Debugf(remote, "Skipping undecryptable file name: %v", err) + return + } + if *cryptShowMapping { + fs.Logf(decryptedRemote, "Encrypts to %q", remote) + } + *entries = append(*entries, f.newObject(obj)) +} + +// Encrypt an directory file name to entries. +func (f *Fs) addDir(entries *fs.DirEntries, dir *fs.Dir) { + remote := dir.Name + decryptedRemote, err := f.cipher.DecryptDirName(remote) + if err != nil { + fs.Debugf(remote, "Skipping undecryptable dir name: %v", err) + return + } + if *cryptShowMapping { + fs.Logf(decryptedRemote, "Encrypts to %q", remote) + } + *entries = append(*entries, f.newDir(dir)) +} + +// Encrypt some directory entries +func (f *Fs) encryptEntries(entries fs.DirEntries) (newEntries fs.DirEntries, err error) { + newEntries = make(fs.DirEntries, 0, len(entries)) + for _, entry := range entries { + switch x := entry.(type) { + case fs.Object: + f.add(&newEntries, x) + case *fs.Dir: + f.addDir(&newEntries, x) + default: + return nil, errors.Errorf("Unknown object type %T", entry) + } + } + return newEntries, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + entries, err = f.Fs.List(f.cipher.EncryptDirName(dir)) + if err != nil { + return nil, err + } + return f.encryptEntries(entries) +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + return f.Fs.Features().ListR(f.cipher.EncryptDirName(dir), func(entries fs.DirEntries) error { + newEntries, err := f.encryptEntries(entries) + if err != nil { + return err + } + return callback(newEntries) + }) } // NewObject finds the Object at remote. @@ -512,84 +593,6 @@ func (o *ObjectInfo) Hash(hash fs.HashType) (string, error) { return "", nil } -// ListOpts wraps a listopts decrypting the directory listing and -// replacing the Objects -type ListOpts struct { - fs.ListOpts - f *Fs - dir string // dir we are listing - mu sync.Mutex // to protect dirs - dirs map[string]struct{} // keep track of synthetic directory objects added -} - -// Make a ListOpts wrapper -func (f *Fs) newListOpts(lo fs.ListOpts, dir string) *ListOpts { - if dir != "" { - dir += "/" - } - return &ListOpts{ - ListOpts: lo, - f: f, - dir: dir, - dirs: make(map[string]struct{}), - } - -} - -// Level gets the recursion level for this listing. -// -// Fses may ignore this, but should implement it for improved efficiency if possible. -// -// Level 1 means list just the contents of the directory -// -// Each returned item must have less than level `/`s in. -func (lo *ListOpts) Level() int { - return lo.ListOpts.Level() -} - -// Add an object to the output. -// If the function returns true, the operation has been aborted. -// Multiple goroutines can safely add objects concurrently. -func (lo *ListOpts) Add(obj fs.Object) (abort bool) { - remote := obj.Remote() - decryptedRemote, err := lo.f.cipher.DecryptFileName(remote) - if err != nil { - fs.Debugf(remote, "Skipping undecryptable file name: %v", err) - return lo.ListOpts.IsFinished() - } - if *cryptShowMapping { - fs.Logf(decryptedRemote, "Encrypts to %q", remote) - } - return lo.ListOpts.Add(lo.f.newObject(obj)) -} - -// AddDir adds a directory to the output. -// If the function returns true, the operation has been aborted. -// Multiple goroutines can safely add objects concurrently. -func (lo *ListOpts) AddDir(dir *fs.Dir) (abort bool) { - remote := dir.Name - decryptedRemote, err := lo.f.cipher.DecryptDirName(remote) - if err != nil { - fs.Debugf(remote, "Skipping undecryptable dir name: %v", err) - return lo.ListOpts.IsFinished() - } - if *cryptShowMapping { - fs.Logf(decryptedRemote, "Encrypts to %q", remote) - } - return lo.ListOpts.AddDir(lo.f.newDir(dir)) -} - -// IncludeDirectory returns whether this directory should be -// included in the listing (and recursed into or not). -func (lo *ListOpts) IncludeDirectory(remote string) bool { - decryptedRemote, err := lo.f.cipher.DecryptDirName(remote) - if err != nil { - fs.Debugf(remote, "Not including undecryptable directory name: %v", err) - return false - } - return lo.ListOpts.IncludeDirectory(decryptedRemote) -} - // Check the interfaces are satisfied var ( _ fs.Fs = (*Fs)(nil) @@ -600,7 +603,7 @@ var ( _ fs.PutUncheckeder = (*Fs)(nil) _ fs.CleanUpper = (*Fs)(nil) _ fs.UnWrapper = (*Fs)(nil) + _ fs.ListRer = (*Fs)(nil) _ fs.ObjectInfo = (*ObjectInfo)(nil) _ fs.Object = (*Object)(nil) - _ fs.ListOpts = (*ListOpts)(nil) ) diff --git a/crypt/crypt2_test.go b/crypt/crypt2_test.go index 6316a5d77..1c059a802 100644 --- a/crypt/crypt2_test.go +++ b/crypt/crypt2_test.go @@ -27,15 +27,20 @@ func TestFsMkdir2(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir2(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty2(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty2(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty2(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound2(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile12(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError2(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile22(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile12(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile22(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile22(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot2(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot2(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir2(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir2(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel22(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel22(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile12(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject2(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and22(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/crypt/crypt3_test.go b/crypt/crypt3_test.go index 8b3d3cb97..605efd480 100644 --- a/crypt/crypt3_test.go +++ b/crypt/crypt3_test.go @@ -27,15 +27,20 @@ func TestFsMkdir3(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir3(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty3(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty3(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty3(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound3(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile13(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError3(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile23(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile13(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile23(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile23(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot3(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot3(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir3(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir3(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel23(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel23(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile13(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject3(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and23(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/crypt/crypt_test.go b/crypt/crypt_test.go index 4f9fcb0f4..613cad365 100644 --- a/crypt/crypt_test.go +++ b/crypt/crypt_test.go @@ -27,15 +27,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/dircache/list.go b/dircache/list.go deleted file mode 100644 index 17068120d..000000000 --- a/dircache/list.go +++ /dev/null @@ -1,82 +0,0 @@ -// Listing utility functions for fses which use dircache - -package dircache - -import ( - "sync" - - "github.com/ncw/rclone/fs" -) - -// ListDirJob describe a directory listing that needs to be done -type ListDirJob struct { - DirID string - Path string - Depth int -} - -// ListDirer describes the interface necessary to use ListDir -type ListDirer interface { - // ListDir reads the directory specified by the job into out, returning any more jobs - ListDir(out fs.ListOpts, job ListDirJob) (jobs []ListDirJob, err error) -} - -// listDir lists the directory using a recursive list from the root -// -// It does this in parallel, calling f.ListDir to do the actual reading -func listDir(f ListDirer, out fs.ListOpts, dirID string, path string) { - // Start some directory listing go routines - var wg sync.WaitGroup // sync closing of go routines - var traversing sync.WaitGroup // running directory traversals - buffer := out.Buffer() - in := make(chan ListDirJob, buffer) - for i := 0; i < buffer; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for job := range in { - jobs, err := f.ListDir(out, job) - if err != nil { - out.SetError(err) - fs.Debugf(f, "Error reading %s: %s", path, err) - } else { - traversing.Add(len(jobs)) - go func() { - // Now we have traversed this directory, send these - // jobs off for traversal in the background - for _, job := range jobs { - in <- job - } - }() - } - traversing.Done() - } - }() - } - - // Start the process - traversing.Add(1) - in <- ListDirJob{DirID: dirID, Path: path, Depth: out.Level() - 1} - traversing.Wait() - close(in) - wg.Wait() -} - -// List walks the path returning iles and directories into out -func (dc *DirCache) List(f ListDirer, out fs.ListOpts, dir string) { - defer out.Finished() - err := dc.FindRoot(false) - if err != nil { - out.SetError(err) - return - } - id, err := dc.FindDir(dir, false) - if err != nil { - out.SetError(err) - return - } - if dir != "" { - dir += "/" - } - listDir(f, out, id, dir) -} diff --git a/drive/drive.go b/drive/drive.go index 765f38c0d..3a8aa0f1d 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -202,17 +202,17 @@ func parseDrivePath(path string) (root string, err error) { return } -// User function to process a File item from listAll +// User function to process a File item from list // // Should return true to finish processing -type listAllFn func(*drive.File) bool +type listFn func(*drive.File) bool // Lists the directory required calling the user function on each item found // // If the user fn ever returns true then it early exits with found = true // // Search params: https://developers.google.com/drive/search-parameters -func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly bool, includeTrashed bool, fn listAllFn) (found bool, err error) { +func (f *Fs) list(dirID string, title string, directoriesOnly bool, filesOnly bool, includeTrashed bool, fn listFn) (found bool, err error) { var query []string if !includeTrashed { @@ -239,7 +239,7 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly if filesOnly { query = append(query, fmt.Sprintf("mimeType!='%s'", driveFolderType)) } - // fmt.Printf("listAll Query = %q\n", query) + // fmt.Printf("list Query = %q\n", query) list := f.svc.Files.List() if len(query) > 0 { list = list.Q(strings.Join(query, " and ")) @@ -488,7 +488,7 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) { // FindLeaf finds a directory of name leaf in the folder with ID pathID func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) { // Find the leaf in pathID - found, err = f.listAll(pathID, leaf, true, false, false, func(item *drive.File) bool { + found, err = f.list(pathID, leaf, true, false, false, func(item *drive.File) bool { if item.Title == leaf { pathIDOut = item.Id return true @@ -554,41 +554,49 @@ func (f *Fs) findExportFormat(filepath string, item *drive.File) (extension, lin return "", "" } -// ListDir reads the directory specified by the job into out, returning any more jobs -func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.ListDirJob, err error) { - fs.Debugf(f, "Reading %q", job.Path) - _, err = f.listAll(job.DirID, "", false, false, false, func(item *drive.File) bool { - remote := job.Path + item.Title +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + err = f.dirCache.FindRoot(false) + if err != nil { + return nil, err + } + directoryID, err := f.dirCache.FindDir(dir, false) + if err != nil { + return nil, err + } + + var iErr error + _, err = f.list(directoryID, "", false, false, false, func(item *drive.File) bool { + remote := path.Join(dir, item.Title) switch { case *driveAuthOwnerOnly && !isAuthOwned(item): // ignore object or directory case item.MimeType == driveFolderType: - if out.IncludeDirectory(remote) { - // cache the directory ID for later lookups - f.dirCache.Put(remote, item.Id) - dir := &fs.Dir{ - Name: remote, - Bytes: -1, - Count: -1, - } - dir.When, _ = time.Parse(timeFormatIn, item.ModifiedDate) - if out.AddDir(dir) { - return true - } - if job.Depth > 0 { - jobs = append(jobs, dircache.ListDirJob{DirID: item.Id, Path: remote + "/", Depth: job.Depth - 1}) - } + // cache the directory ID for later lookups + f.dirCache.Put(remote, item.Id) + d := &fs.Dir{ + Name: remote, + Bytes: -1, + Count: -1, } + d.When, _ = time.Parse(timeFormatIn, item.ModifiedDate) + entries = append(entries, d) case item.Md5Checksum != "" || item.FileSize > 0: // If item has MD5 sum or a length it is a file stored on drive o, err := f.newObjectWithInfo(remote, item) if err != nil { - out.SetError(err) - return true - } - if out.Add(o) { + iErr = err return true } + entries = append(entries, o) case len(item.ExportLinks) != 0: // If item has export links then it is a google doc extension, link := f.findExportFormat(remote, item) @@ -597,7 +605,7 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache. } else { o, err := f.newObjectWithInfo(remote+"."+extension, item) if err != nil { - out.SetError(err) + iErr = err return true } if !*driveSkipGdocs { @@ -605,9 +613,7 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache. obj.isDocument = true obj.url = link obj.bytes = -1 - if out.Add(o) { - return true - } + entries = append(entries, o) } else { fs.Debugf(f, "Skip google document: %q", remote) } @@ -617,13 +623,13 @@ func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache. } return false }) - fs.Debugf(f, "Finished reading %q", job.Path) - return jobs, err -} - -// List walks the path returning files and directories to out -func (f *Fs) List(out fs.ListOpts, dir string) { - f.dirCache.List(f, out, dir) + if err != nil { + return nil, err + } + if iErr != nil { + return nil, iErr + } + return entries, nil } // Creates a drive.File info from the parameters passed in and a half @@ -731,7 +737,7 @@ func (f *Fs) Rmdir(dir string) error { return err } var trashedFiles = false - found, err := f.listAll(directoryID, "", false, false, true, func(item *drive.File) bool { + found, err := f.list(directoryID, "", false, false, true, func(item *drive.File) bool { if item.Labels == nil || !item.Labels.Trashed { fs.Debugf(dir, "Rmdir: contains file: %q", item.Title) return true @@ -1145,7 +1151,7 @@ func (o *Object) readMetaData() (err error) { return err } - found, err := o.fs.listAll(directoryID, leaf, false, true, false, func(item *drive.File) bool { + found, err := o.fs.list(directoryID, leaf, false, true, false, func(item *drive.File) bool { if item.Title == leaf { o.setMetaData(item) return true diff --git a/drive/drive_test.go b/drive/drive_test.go index 098d94937..4f48a2206 100644 --- a/drive/drive_test.go +++ b/drive/drive_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/dropbox/dropbox.go b/dropbox/dropbox.go index 2d6696e8d..da67fbeeb 100644 --- a/dropbox/dropbox.go +++ b/dropbox/dropbox.go @@ -5,9 +5,9 @@ package dropbox // FIXME dropbox for business would be quite easy to add /* -FIXME is case folding of PathDisplay going to cause a problem? +The Case folding of PathDisplay problem -From the docs +From the docs: path_display String. The cased path to be used for display purposes only. In rare instances the casing will not correctly match the user's @@ -17,8 +17,7 @@ casing. Changes to only the casing of paths won't be returned by list_folder/continue. This field will be null if the file or folder is not mounted. This field is optional. -This only becomes a problem if dropbox implements the ListR interface -which it currently doesn't. +We solve this by not implementing the ListR interface. The dropbox remote will recurse directory by directory and all will be well. */ import ( @@ -315,8 +314,16 @@ func (f *Fs) stripRoot(path string) (string, error) { return strip(path, f.slashRootSlash) } -// Walk the root returning a channel of Objects -func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) { +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { root := f.slashRoot if dir != "" { root += "/" + dir @@ -324,12 +331,11 @@ func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) { started := false var res *files.ListFolderResult - var err error for { if !started { arg := files.ListFolderArg{ Path: root, - Recursive: recursive, + Recursive: false, } if root == "/" { arg.Path = "" // Specify root folder as empty string @@ -346,8 +352,7 @@ func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) { err = fs.ErrorDirNotFound } } - out.SetError(err) - return + return nil, err } started = false } else { @@ -359,8 +364,7 @@ func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) { return shouldRetry(err) }) if err != nil { - out.SetError(errors.Wrap(err, "list continue")) - return + return nil, errors.Wrap(err, "list continue") } } for _, entry := range res.Entries { @@ -384,56 +388,36 @@ func (f *Fs) list(out fs.ListOpts, dir string, recursive bool) { if folderInfo != nil { name, err := f.stripRoot(entryPath + "/") if err != nil { - out.SetError(err) - return + return nil, err } name = strings.Trim(name, "/") if name != "" && name != dir { - dir := &fs.Dir{ + d := &fs.Dir{ Name: name, When: time.Now(), //When: folderInfo.ClientMtime, //Bytes: folderInfo.Bytes, //Count: -1, } - if out.AddDir(dir) { - return - } + entries = append(entries, d) } } else if fileInfo != nil { path, err := f.stripRoot(entryPath) if err != nil { - out.SetError(err) - return + return nil, err } o, err := f.newObjectWithInfo(path, fileInfo) if err != nil { - out.SetError(err) - return - } - if out.Add(o) { - return + return nil, err } + entries = append(entries, o) } } if !res.HasMore { break } } -} - -// List walks the path returning a channel of Objects -func (f *Fs) List(out fs.ListOpts, dir string) { - defer out.Finished() - level := out.Level() - switch level { - case 1: - f.list(out, dir, false) - case fs.MaxLevel: - f.list(out, dir, true) - default: - out.SetError(fs.ErrorLevelNotSupported) - } + return entries, nil } // A read closer which doesn't close the input diff --git a/dropbox/dropbox_test.go b/dropbox/dropbox_test.go index 084780db5..3d79893ac 100644 --- a/dropbox/dropbox_test.go +++ b/dropbox/dropbox_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/fs/fs.go b/fs/fs.go index 57777dd9d..6bc6983e2 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -41,7 +41,7 @@ var ( ErrorObjectNotFound = errors.New("object not found") ErrorLevelNotSupported = errors.New("level value not supported") ErrorListAborted = errors.New("list aborted") - ErrorListOnlyRoot = errors.New("can only list from root") + ErrorListBucketRequired = errors.New("bucket or container name is needed in remote") ErrorIsFile = errors.New("is a file not a directory") ErrorNotAFile = errors.New("is a not a regular file") ErrorNotDeleting = errors.New("not deleting files as there were IO errors") @@ -103,17 +103,16 @@ func Register(info *RegInfo) { // ListFser is the interface for listing a remote Fs type ListFser interface { - // List the objects and directories of the Fs starting from dir + // List the objects and directories in dir into entries. The + // entries can be returned in any order but should be for a + // complete directory. // - // dir should be "" to start from the root, and should not - // have trailing slashes. + // dir should be "" to list the root, and should not have + // trailing slashes. // - // This should return ErrDirNotFound (using out.SetError()) - // if the directory isn't found. - // - // Fses must support recursion levels of fs.MaxLevel and 1. - // They may return ErrorLevelNotSupported otherwise. - List(out ListOpts, dir string) + // This should return ErrDirNotFound if the directory isn't + // found. + List(dir string) (entries DirEntries, err error) // NewObject finds the Object at remote. If it can't be found // it returns the error ErrorObjectNotFound. @@ -220,6 +219,15 @@ type MimeTyper interface { MimeType() string } +// ListRCallback defines a callback function for ListR to use +// +// It is called for each tranche of entries read from the listing and +// if it returns an error, the listing stops. +type ListRCallback func(entries DirEntries) error + +// ListRFn is defines the call used to recursively list a directory +type ListRFn func(dir string, callback ListRCallback) error + // Features describe the optional features of the Fs type Features struct { // Feature flags @@ -302,12 +310,17 @@ type Features struct { // dir should be "" to start from the root, and should not // have trailing slashes. // - // This should return ErrDirNotFound (using out.SetError()) - // if the directory isn't found. + // This should return ErrDirNotFound if the directory isn't + // found. + // + // It should call callback for each tranche of entries read. + // These need not be returned in any particular order. If + // callback returns an error then the listing will stop + // immediately. // // Don't implement this unless you have a more efficient way // of listing recursively that doing a directory traversal. - ListR func(out ListOpts, dir string) + ListR ListRFn } // Fill fills in the function pointers in the Features struct from the @@ -506,54 +519,22 @@ type ListRer interface { // dir should be "" to start from the root, and should not // have trailing slashes. // - // This should return ErrDirNotFound (using out.SetError()) - // if the directory isn't found. + // This should return ErrDirNotFound if the directory isn't + // found. + // + // It should call callback for each tranche of entries read. + // These need not be returned in any particular order. If + // callback returns an error then the listing will stop + // immediately. // // Don't implement this unless you have a more efficient way // of listing recursively that doing a directory traversal. - ListR(out ListOpts, dir string) + ListR(dir string, callback ListRCallback) error } // ObjectsChan is a channel of Objects type ObjectsChan chan Object -// ListOpts describes the interface used for Fs.List operations -type ListOpts interface { - // Add an object to the output. - // If the function returns true, the operation has been aborted. - // Multiple goroutines can safely add objects concurrently. - Add(obj Object) (abort bool) - - // Add a directory to the output. - // If the function returns true, the operation has been aborted. - // Multiple goroutines can safely add objects concurrently. - AddDir(dir *Dir) (abort bool) - - // IncludeDirectory returns whether this directory should be - // included in the listing (and recursed into or not). - IncludeDirectory(remote string) bool - - // SetError will set an error state, and will cause the listing to - // be aborted. - // Multiple goroutines can set the error state concurrently, - // but only the first will be returned to the caller. - SetError(err error) - - // Level returns the level it should recurse to. Fses may - // ignore this in which case the listing will be less - // efficient. - Level() int - - // Buffer returns the channel depth in use - Buffer() int - - // Finished should be called when listing is finished - Finished() - - // IsFinished returns whether Finished or SetError have been called - IsFinished() bool -} - // Objects is a slice of Object~s type Objects []Object diff --git a/fs/lister.go b/fs/lister.go deleted file mode 100644 index 1d4ffcf2e..000000000 --- a/fs/lister.go +++ /dev/null @@ -1,314 +0,0 @@ -// This file implements the Lister object - -package fs - -import "sync" - -// listerResult is returned by the lister methods -type listerResult struct { - Obj Object - Dir *Dir - Err error -} - -// Lister objects are used for controlling listing of Fs objects -type Lister struct { - mu sync.RWMutex - buffer int - abort bool - results chan listerResult - closeOnce sync.Once - level int - filter *Filter - err error -} - -// NewLister creates a Lister object. -// -// The default channel buffer size will be Config.Checkers unless -// overridden with SetBuffer. The default level will be infinite. -func NewLister() *Lister { - o := &Lister{} - return o.SetLevel(-1).SetBuffer(Config.Checkers) -} - -// Finds and lists the files passed in -// -// Note we ignore the dir and just return all the files in the list -func (o *Lister) listFiles(f ListFser, dir string, files FilesMap) { - buffer := o.Buffer() - jobs := make(chan string, buffer) - var wg sync.WaitGroup - - // Start some listing go routines so we find those name in parallel - wg.Add(buffer) - for i := 0; i < buffer; i++ { - go func() { - defer wg.Done() - for remote := range jobs { - obj, err := f.NewObject(remote) - if err == ErrorObjectNotFound { - // silently ignore files that aren't found in the files list - } else if err != nil { - o.SetError(err) - } else { - o.Add(obj) - } - } - }() - } - - // Pump the names in - for name := range files { - jobs <- name - if o.IsFinished() { - break - } - } - close(jobs) - wg.Wait() - - // Signal that this listing is over - o.Finished() -} - -// Start starts a go routine listing the Fs passed in. It returns the -// same Lister that was passed in for convenience. -func (o *Lister) Start(f ListFser, dir string) *Lister { - o.results = make(chan listerResult, o.buffer) - if o.filter != nil && o.filter.Files() != nil { - go o.listFiles(f, dir, o.filter.Files()) - } else { - go f.List(o, dir) - } - return o -} - -// SetLevel sets the level to recurse to. It returns same Lister that -// was passed in for convenience. If Level is < 0 then it sets it to -// infinite. Must be called before Start(). -func (o *Lister) SetLevel(level int) *Lister { - if level < 0 { - o.level = MaxLevel - } else { - o.level = level - } - return o -} - -// SetFilter sets the Filter that is in use. It defaults to no -// filtering. Must be called before Start(). -func (o *Lister) SetFilter(filter *Filter) *Lister { - o.filter = filter - return o -} - -// Level gets the recursion level for this listing. -// -// Fses may ignore this, but should implement it for improved efficiency if possible. -// -// Level 1 means list just the contents of the directory -// -// Each returned item must have less than level `/`s in. -func (o *Lister) Level() int { - return o.level -} - -// SetBuffer sets the channel buffer size in use. Must be called -// before Start(). -func (o *Lister) SetBuffer(buffer int) *Lister { - if buffer < 1 { - buffer = 1 - } - o.buffer = buffer - return o -} - -// Buffer gets the channel buffer size in use -func (o *Lister) Buffer() int { - return o.buffer -} - -// Add an object to the output. -// If the function returns true, the operation has been aborted. -// Multiple goroutines can safely add objects concurrently. -func (o *Lister) Add(obj Object) (abort bool) { - o.mu.RLock() - defer o.mu.RUnlock() - if o.abort { - return true - } - o.results <- listerResult{Obj: obj} - return false -} - -// AddDir will a directory to the output. -// If the function returns true, the operation has been aborted. -// Multiple goroutines can safely add objects concurrently. -func (o *Lister) AddDir(dir *Dir) (abort bool) { - o.mu.RLock() - defer o.mu.RUnlock() - if o.abort { - return true - } - o.results <- listerResult{Dir: dir} - return false -} - -// Error returns a globally application error that's been set on the Lister -// object. -func (o *Lister) Error() error { - o.mu.RLock() - defer o.mu.RUnlock() - return o.err -} - -// IncludeDirectory returns whether this directory should be -// included in the listing (and recursed into or not). -func (o *Lister) IncludeDirectory(remote string) bool { - if o.filter == nil { - return true - } - return o.filter.IncludeDirectory(remote) -} - -// finished closes the results channel and sets abort - must be called -// with o.mu held. -func (o *Lister) finished() { - o.closeOnce.Do(func() { - close(o.results) - o.abort = true - }) -} - -// SetError will set an error state, and will cause the listing to -// be aborted. -// Multiple goroutines can set the error state concurrently, -// but only the first will be returned to the caller. -func (o *Lister) SetError(err error) { - o.mu.Lock() - if err != nil && !o.abort { - o.err = err - o.results <- listerResult{Err: err} - o.finished() - } - o.mu.Unlock() -} - -// Finished should be called when listing is finished -func (o *Lister) Finished() { - o.mu.Lock() - o.finished() - o.mu.Unlock() -} - -// IsFinished returns whether the directory listing is finished or not -func (o *Lister) IsFinished() bool { - o.mu.RLock() - defer o.mu.RUnlock() - return o.abort -} - -// Get an object from the listing. -// Will return either an object or a directory, never both. -// Will return (nil, nil, nil) when all objects have been returned. -func (o *Lister) Get() (Object, *Dir, error) { - select { - case r := <-o.results: - return r.Obj, r.Dir, r.Err - } -} - -// GetAll gets all the objects and dirs from the listing. -func (o *Lister) GetAll() (objs []Object, dirs []*Dir, err error) { - for { - obj, dir, err := o.Get() - switch { - case err != nil: - return nil, nil, err - case obj != nil: - objs = append(objs, obj) - case dir != nil: - dirs = append(dirs, dir) - default: - return objs, dirs, nil - } - } -} - -// GetObject will return an object from the listing. -// It will skip over any directories. -// Will return (nil, nil) when all objects have been returned. -func (o *Lister) GetObject() (Object, error) { - for { - obj, dir, err := o.Get() - switch { - case err != nil: - return nil, err - case obj != nil: - return obj, nil - case dir != nil: - // ignore - default: - return nil, nil - } - } -} - -// GetObjects will return a slice of object from the listing. -// It will skip over any directories. -func (o *Lister) GetObjects() (objs []Object, err error) { - for { - obj, dir, err := o.Get() - switch { - case err != nil: - return nil, err - case obj != nil: - objs = append(objs, obj) - case dir != nil: - // ignore - default: - return objs, nil - } - } -} - -// GetDir will return a directory from the listing. -// It will skip over any objects. -// Will return (nil, nil) when all objects have been returned. -func (o *Lister) GetDir() (*Dir, error) { - for { - obj, dir, err := o.Get() - switch { - case err != nil: - return nil, err - case obj != nil: - // ignore - case dir != nil: - return dir, nil - default: - return nil, nil - } - } -} - -// GetDirs will return a slice of directories from the listing. -// It will skip over any objects. -func (o *Lister) GetDirs() (dirs []*Dir, err error) { - for { - obj, dir, err := o.Get() - switch { - case err != nil: - return nil, err - case obj != nil: - // ignore - case dir != nil: - dirs = append(dirs, dir) - default: - return dirs, nil - } - } -} - -// Check interface -var _ ListOpts = (*Lister)(nil) diff --git a/fs/lister_test.go b/fs/lister_test.go deleted file mode 100644 index 6fb97635c..000000000 --- a/fs/lister_test.go +++ /dev/null @@ -1,384 +0,0 @@ -package fs - -import ( - "io" - "sort" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestListerNew(t *testing.T) { - o := NewLister() - assert.Equal(t, Config.Checkers, o.buffer) - assert.Equal(t, false, o.abort) - assert.Equal(t, MaxLevel, o.level) -} - -var errNotImpl = errors.New("not implemented") - -type mockObject string - -func (o mockObject) String() string { return string(o) } -func (o mockObject) Fs() Info { return nil } -func (o mockObject) Remote() string { return string(o) } -func (o mockObject) Hash(HashType) (string, error) { return "", errNotImpl } -func (o mockObject) ModTime() (t time.Time) { return t } -func (o mockObject) Size() int64 { return 0 } -func (o mockObject) Storable() bool { return true } -func (o mockObject) SetModTime(time.Time) error { return errNotImpl } -func (o mockObject) Open(options ...OpenOption) (io.ReadCloser, error) { return nil, errNotImpl } -func (o mockObject) Update(in io.Reader, src ObjectInfo, options ...OpenOption) error { - return errNotImpl -} -func (o mockObject) Remove() error { return errNotImpl } - -type mockFs struct { - listFn func(o ListOpts, dir string) -} - -func (f *mockFs) List(o ListOpts, dir string) { - defer o.Finished() - f.listFn(o, dir) -} - -func (f *mockFs) NewObject(remote string) (Object, error) { - return mockObject(remote), nil -} - -func TestListerStart(t *testing.T) { - f := &mockFs{} - ranList := false - f.listFn = func(o ListOpts, dir string) { - ranList = true - } - o := NewLister().Start(f, "") - objs, dirs, err := o.GetAll() - require.Nil(t, err) - assert.Len(t, objs, 0) - assert.Len(t, dirs, 0) - assert.Equal(t, true, ranList) -} - -func TestListerStartWithFiles(t *testing.T) { - f := &mockFs{} - ranList := false - f.listFn = func(o ListOpts, dir string) { - ranList = true - } - filter, err := NewFilter() - require.NoError(t, err) - wantNames := []string{"potato", "sausage", "rutabaga", "carrot", "lettuce"} - sort.Strings(wantNames) - for _, name := range wantNames { - err = filter.AddFile(name) - require.NoError(t, err) - } - o := NewLister().SetFilter(filter).Start(f, "") - objs, dirs, err := o.GetAll() - require.Nil(t, err) - assert.Len(t, dirs, 0) - assert.Equal(t, false, ranList) - var gotNames []string - for _, obj := range objs { - gotNames = append(gotNames, obj.Remote()) - } - sort.Strings(gotNames) - assert.Equal(t, wantNames, gotNames) -} - -func TestListerSetLevel(t *testing.T) { - o := NewLister() - o.SetLevel(1) - assert.Equal(t, 1, o.level) - o.SetLevel(0) - assert.Equal(t, 0, o.level) - o.SetLevel(-1) - assert.Equal(t, MaxLevel, o.level) -} - -func TestListerSetFilter(t *testing.T) { - filter := &Filter{} - o := NewLister().SetFilter(filter) - assert.Equal(t, filter, o.filter) -} - -func TestListerLevel(t *testing.T) { - o := NewLister() - assert.Equal(t, MaxLevel, o.Level()) - o.SetLevel(123) - assert.Equal(t, 123, o.Level()) -} - -func TestListerSetBuffer(t *testing.T) { - o := NewLister() - o.SetBuffer(2) - assert.Equal(t, 2, o.buffer) - o.SetBuffer(1) - assert.Equal(t, 1, o.buffer) - o.SetBuffer(0) - assert.Equal(t, 1, o.buffer) - o.SetBuffer(-1) - assert.Equal(t, 1, o.buffer) -} - -func TestListerBuffer(t *testing.T) { - o := NewLister() - assert.Equal(t, Config.Checkers, o.Buffer()) - o.SetBuffer(123) - assert.Equal(t, 123, o.Buffer()) -} - -func TestListerAdd(t *testing.T) { - f := &mockFs{} - objs := []Object{ - mockObject("1"), - mockObject("2"), - } - f.listFn = func(o ListOpts, dir string) { - for _, obj := range objs { - assert.Equal(t, false, o.Add(obj)) - } - } - o := NewLister().Start(f, "") - gotObjs, gotDirs, err := o.GetAll() - require.Nil(t, err) - assert.Equal(t, objs, gotObjs) - assert.Len(t, gotDirs, 0) -} - -func TestListerAddDir(t *testing.T) { - f := &mockFs{} - dirs := []*Dir{ - &Dir{Name: "1"}, - &Dir{Name: "2"}, - } - f.listFn = func(o ListOpts, dir string) { - for _, dir := range dirs { - assert.Equal(t, false, o.AddDir(dir)) - } - } - o := NewLister().Start(f, "") - gotObjs, gotDirs, err := o.GetAll() - require.Nil(t, err) - assert.Len(t, gotObjs, 0) - assert.Equal(t, dirs, gotDirs) -} - -func TestListerIncludeDirectory(t *testing.T) { - o := NewLister() - assert.Equal(t, true, o.IncludeDirectory("whatever")) - filter, err := NewFilter() - require.Nil(t, err) - require.NotNil(t, filter) - require.Nil(t, filter.AddRule("!")) - require.Nil(t, filter.AddRule("+ potato/*")) - require.Nil(t, filter.AddRule("- *")) - o.SetFilter(filter) - assert.Equal(t, false, o.IncludeDirectory("floop")) - assert.Equal(t, true, o.IncludeDirectory("potato")) - assert.Equal(t, false, o.IncludeDirectory("potato/sausage")) -} - -func TestListerSetError(t *testing.T) { - f := &mockFs{} - f.listFn = func(o ListOpts, dir string) { - assert.Equal(t, false, o.Add(mockObject("1"))) - o.SetError(errNotImpl) - assert.Equal(t, true, o.Add(mockObject("2"))) - o.SetError(errors.New("not signalled")) - assert.Equal(t, true, o.AddDir(&Dir{Name: "2"})) - } - o := NewLister().Start(f, "") - gotObjs, gotDirs, err := o.GetAll() - assert.Equal(t, err, errNotImpl) - assert.Nil(t, gotObjs) - assert.Nil(t, gotDirs) -} - -func TestListerIsFinished(t *testing.T) { - f := &mockFs{} - f.listFn = func(o ListOpts, dir string) { - assert.Equal(t, false, o.IsFinished()) - o.Finished() - assert.Equal(t, true, o.IsFinished()) - } - o := NewLister().Start(f, "") - gotObjs, gotDirs, err := o.GetAll() - assert.Nil(t, err) - assert.Len(t, gotObjs, 0) - assert.Len(t, gotDirs, 0) -} - -func testListerGet(t *testing.T) *Lister { - f := &mockFs{} - f.listFn = func(o ListOpts, dir string) { - assert.Equal(t, false, o.Add(mockObject("1"))) - assert.Equal(t, false, o.AddDir(&Dir{Name: "2"})) - } - return NewLister().Start(f, "") -} - -func TestListerGet(t *testing.T) { - o := testListerGet(t) - obj, dir, err := o.Get() - assert.Nil(t, err) - assert.Equal(t, obj.Remote(), "1") - assert.Nil(t, dir) - obj, dir, err = o.Get() - assert.Nil(t, err) - assert.Nil(t, obj) - assert.Equal(t, dir.Name, "2") - obj, dir, err = o.Get() - assert.Nil(t, err) - assert.Nil(t, obj) - assert.Nil(t, dir) - assert.Equal(t, true, o.IsFinished()) -} - -func TestListerGetObject(t *testing.T) { - o := testListerGet(t) - obj, err := o.GetObject() - assert.Nil(t, err) - assert.Equal(t, obj.Remote(), "1") - obj, err = o.GetObject() - assert.Nil(t, err) - assert.Nil(t, obj) - assert.Equal(t, true, o.IsFinished()) -} - -func TestListerGetDir(t *testing.T) { - o := testListerGet(t) - dir, err := o.GetDir() - assert.Nil(t, err) - assert.Equal(t, dir.Name, "2") - dir, err = o.GetDir() - assert.Nil(t, err) - assert.Nil(t, dir) - assert.Equal(t, true, o.IsFinished()) -} - -func testListerGetError(t *testing.T) *Lister { - f := &mockFs{} - f.listFn = func(o ListOpts, dir string) { - o.SetError(errNotImpl) - } - return NewLister().Start(f, "") -} - -func TestListerGetError(t *testing.T) { - o := testListerGetError(t) - obj, dir, err := o.Get() - assert.Equal(t, err, errNotImpl) - assert.Nil(t, obj) - assert.Nil(t, dir) - obj, dir, err = o.Get() - assert.Nil(t, err) - assert.Nil(t, obj) - assert.Nil(t, dir) - assert.Equal(t, true, o.IsFinished()) -} - -func TestListerGetObjectError(t *testing.T) { - o := testListerGetError(t) - obj, err := o.GetObject() - assert.Equal(t, err, errNotImpl) - assert.Nil(t, obj) - obj, err = o.GetObject() - assert.Nil(t, err) - assert.Nil(t, obj) - assert.Equal(t, true, o.IsFinished()) -} - -func TestListerGetDirError(t *testing.T) { - o := testListerGetError(t) - dir, err := o.GetDir() - assert.Equal(t, err, errNotImpl) - assert.Nil(t, dir) - dir, err = o.GetDir() - assert.Nil(t, err) - assert.Nil(t, dir) - assert.Equal(t, true, o.IsFinished()) -} - -func testListerGetAll(t *testing.T) (*Lister, []Object, []*Dir) { - objs := []Object{ - mockObject("1f"), - mockObject("2f"), - mockObject("3f"), - } - dirs := []*Dir{ - &Dir{Name: "1d"}, - &Dir{Name: "2d"}, - } - f := &mockFs{} - f.listFn = func(o ListOpts, dir string) { - assert.Equal(t, false, o.Add(objs[0])) - assert.Equal(t, false, o.Add(objs[1])) - assert.Equal(t, false, o.AddDir(dirs[0])) - assert.Equal(t, false, o.Add(objs[2])) - assert.Equal(t, false, o.AddDir(dirs[1])) - } - return NewLister().Start(f, ""), objs, dirs -} - -func TestListerGetAll(t *testing.T) { - o, objs, dirs := testListerGetAll(t) - gotObjs, gotDirs, err := o.GetAll() - assert.Nil(t, err) - assert.Equal(t, objs, gotObjs) - assert.Equal(t, dirs, gotDirs) - assert.Equal(t, true, o.IsFinished()) -} - -func TestListerGetObjects(t *testing.T) { - o, objs, _ := testListerGetAll(t) - gotObjs, err := o.GetObjects() - assert.Nil(t, err) - assert.Equal(t, objs, gotObjs) - assert.Equal(t, true, o.IsFinished()) -} - -func TestListerGetDirs(t *testing.T) { - o, _, dirs := testListerGetAll(t) - gotDirs, err := o.GetDirs() - assert.Nil(t, err) - assert.Equal(t, dirs, gotDirs) - assert.Equal(t, true, o.IsFinished()) -} - -func testListerGetAllError(t *testing.T) *Lister { - f := &mockFs{} - f.listFn = func(o ListOpts, dir string) { - o.SetError(errNotImpl) - } - return NewLister().Start(f, "") -} - -func TestListerGetAllError(t *testing.T) { - o := testListerGetAllError(t) - gotObjs, gotDirs, err := o.GetAll() - assert.Equal(t, errNotImpl, err) - assert.Len(t, gotObjs, 0) - assert.Len(t, gotDirs, 0) - assert.Equal(t, true, o.IsFinished()) -} - -func TestListerGetObjectsError(t *testing.T) { - o := testListerGetAllError(t) - gotObjs, err := o.GetObjects() - assert.Equal(t, errNotImpl, err) - assert.Len(t, gotObjs, 0) - assert.Equal(t, true, o.IsFinished()) -} - -func TestListerGetDirsError(t *testing.T) { - o := testListerGetAllError(t) - gotDirs, err := o.GetDirs() - assert.Equal(t, errNotImpl, err) - assert.Len(t, gotDirs, 0) - assert.Equal(t, true, o.IsFinished()) -} diff --git a/fs/operations.go b/fs/operations.go index b624ebc75..b8f1e83b7 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -595,41 +595,41 @@ func (ds DirEntries) ForDirError(fn func(dir *Dir) error) error { // dir is the start directory, "" for root // // If includeAll is specified all files will be added, otherwise only -// files passing the filter will be added. +// files and directories passing the filter will be added. // // Files will be returned in sorted order func ListDirSorted(fs Fs, includeAll bool, dir string) (entries DirEntries, err error) { - list := NewLister().SetLevel(1) - if !includeAll { - list.SetFilter(Config.Filter) - } - list.Start(fs, dir) - for { - o, dir, err := list.Get() - if err != nil { - return nil, err - } else if o != nil { - // Make sure we don't delete excluded files if not required - if includeAll || Config.Filter.IncludeObject(o) { - entries = append(entries, o) - } else { - Debugf(o, "Excluded from sync (and deletion)") - } - } else if dir != nil { - if includeAll || Config.Filter.IncludeDirectory(dir.Remote()) { - entries = append(entries, dir) - } else { - Debugf(dir, "Excluded from sync (and deletion)") - } - } else { - // finishd since err, o, dir == nil - break - } - } - err = list.Error() + // Get unfiltered entries from the fs + entries, err = fs.List(dir) if err != nil { return nil, err } + + // filter the entries if required + if !includeAll { + newEntries := make(DirEntries, 0, len(entries)) + for _, entry := range entries { + switch x := entry.(type) { + case Object: + // Make sure we don't delete excluded files if not required + if Config.Filter.IncludeObject(x) { + newEntries = append(newEntries, entry) + } else { + Debugf(x, "Excluded from sync (and deletion)") + } + case *Dir: + if Config.Filter.IncludeDirectory(x.Remote()) { + newEntries = append(newEntries, entry) + } else { + Debugf(x, "Excluded from sync (and deletion)") + } + default: + return nil, errors.Errorf("unknown object type %T", entry) + } + } + entries = newEntries + } + // sort the directory entries by Remote sort.Sort(entries) return entries, nil diff --git a/fs/walk.go b/fs/walk.go index ac6b97743..509dead44 100644 --- a/fs/walk.go +++ b/fs/walk.go @@ -71,6 +71,10 @@ func WalkN(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error // It implements Walk using recursive directory listing if // available, or returns ErrorCantListR if not. func WalkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error { + listR := f.Features().ListR + if listR == nil { + return ErrorCantListR + } return walkR(f, path, includeAll, maxLevel, fn, listR) } @@ -254,57 +258,12 @@ func (dt DirTree) String() string { return out.String() } -type listRCallback func(entries DirEntries) error - -type listRFunc func(f Fs, dir string, callback listRCallback) error - -// FIXME Pretend ListR function -func listR(f Fs, dir string, callback listRCallback) (err error) { - listR := f.Features().ListR - if listR == nil { - return ErrorCantListR - } - const maxEntries = 100 - entries := make(DirEntries, 0, maxEntries) - list := NewLister() - list.Start(f, dir) - for { - o, dir, err := list.Get() - if err != nil { - return err - } else if o != nil { - entries = append(entries, o) - } else if dir != nil { - entries = append(entries, dir) - } else { - // finishd since err, o, dir == nil - break - } - if len(entries) >= maxEntries { - err = callback(entries) - if err != nil { - return err - } - entries = entries[:0] - } - } - err = list.Error() - if err != nil { - return err - } - if len(entries) > 0 { - err = callback(entries) - if err != nil { - return err - } - } - return nil - -} - -func walkRDirTree(f Fs, path string, includeAll bool, maxLevel int, listRFn listRFunc) (DirTree, error) { +func walkRDirTree(f Fs, path string, includeAll bool, maxLevel int, listR ListRFn) (DirTree, error) { dirs := make(DirTree) - err := listRFn(f, path, func(entries DirEntries) error { + var mu sync.Mutex + err := listR(path, func(entries DirEntries) error { + mu.Lock() + defer mu.Unlock() for _, entry := range entries { slashes := strings.Count(entry.Remote(), "/") switch x := entry.(type) { @@ -352,13 +311,19 @@ func walkRDirTree(f Fs, path string, includeAll bool, maxLevel int, listRFn list return dirs, nil } -// NewDirTree returns a DirTree filled with the directory listing using the parameters supplied +// NewDirTree returns a DirTree filled with the directory listing +// using the parameters supplied. This will return ErrorCantListR for +// remotes which don't support ListR. func NewDirTree(f Fs, path string, includeAll bool, maxLevel int) (DirTree, error) { + listR := f.Features().ListR + if listR == nil { + return nil, ErrorCantListR + } return walkRDirTree(f, path, includeAll, maxLevel, listR) } -func walkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc, listRFn listRFunc) error { - dirs, err := walkRDirTree(f, path, includeAll, maxLevel, listRFn) +func walkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc, listR ListRFn) error { + dirs, err := walkRDirTree(f, path, includeAll, maxLevel, listR) if err != nil { return err } @@ -410,3 +375,41 @@ func WalkGetAll(f Fs, path string, includeAll bool, maxLevel int) (objs []Object }) return } + +// ListRHelper is used in the implementation of ListR to accumulate DirEntries +type ListRHelper struct { + callback ListRCallback + entries DirEntries +} + +// NewListRHelper should be called from ListR with the callback passed in +func NewListRHelper(callback ListRCallback) *ListRHelper { + return &ListRHelper{ + callback: callback, + } +} + +// send sends the stored entries to the callback if there are >= max +// entries. +func (lh *ListRHelper) send(max int) (err error) { + if len(lh.entries) >= max { + err = lh.callback(lh.entries) + lh.entries = lh.entries[:0] + } + return err +} + +// Add an entry to the stored entries and send them if there are more +// than a certain amount +func (lh *ListRHelper) Add(entry BasicInfo) error { + if entry == nil { + return nil + } + lh.entries = append(lh.entries, entry) + return lh.send(100) +} + +// Flush the stored entries (if any) sending them to the callback +func (lh *ListRHelper) Flush() error { + return lh.send(1) +} diff --git a/fs/walk_test.go b/fs/walk_test.go index 1ee2461b9..4a13f1fef 100644 --- a/fs/walk_test.go +++ b/fs/walk_test.go @@ -2,8 +2,10 @@ package fs import ( "fmt" + "io" "sync" "testing" + "time" "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -34,6 +36,24 @@ type ( } ) +var errNotImpl = errors.New("not implemented") + +type mockObject string + +func (o mockObject) String() string { return string(o) } +func (o mockObject) Fs() Info { return nil } +func (o mockObject) Remote() string { return string(o) } +func (o mockObject) Hash(HashType) (string, error) { return "", errNotImpl } +func (o mockObject) ModTime() (t time.Time) { return t } +func (o mockObject) Size() int64 { return 0 } +func (o mockObject) Storable() bool { return true } +func (o mockObject) SetModTime(time.Time) error { return errNotImpl } +func (o mockObject) Open(options ...OpenOption) (io.ReadCloser, error) { return nil, errNotImpl } +func (o mockObject) Update(in io.Reader, src ObjectInfo, options ...OpenOption) error { + return errNotImpl +} +func (o mockObject) Remove() error { return errNotImpl } + func newListDirs(t *testing.T, f Fs, includeAll bool, results listResults, walkErrors errorMap, finalError error) *listDirs { return &listDirs{ t: t, @@ -82,11 +102,9 @@ func (ls *listDirs) ListDir(f Fs, includeAll bool, dir string) (entries DirEntri } // ListR returns the expected listing for the directory using ListR -func (ls *listDirs) ListR(f Fs, dir string, callback listRCallback) (err error) { +func (ls *listDirs) ListR(dir string, callback ListRCallback) (err error) { ls.mu.Lock() defer ls.mu.Unlock() - assert.Equal(ls.t, ls.fs, f) - //assert.Equal(ls.t, ls.includeAll, includeAll) var errorReturn error for dirPath, result := range ls.results { @@ -392,8 +410,8 @@ func TestWalkMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() } func TestWalkRMultiErrors(t *testing.T) { testWalkMultiErrors(t).Walk() } // a very simple listRcallback function -func makeListRCallback(entries DirEntries, err error) listRFunc { - return func(f Fs, dir string, callback listRCallback) error { +func makeListRCallback(entries DirEntries, err error) ListRFn { + return func(dir string, callback ListRCallback) error { if err == nil { err = callback(entries) } diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index d064fc02c..fd996e89d 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -99,6 +99,20 @@ func skipIfNotOk(t *testing.T) { } } +// Skip if remote is not ListR capable, otherwise set the useListR +// flag, returning a function to restore its value +func skipIfNotListR(t *testing.T) func() { + skipIfNotOk(t) + if remote.Features().ListR == nil { + t.Skip("FS has no ListR interface") + } + previous := fs.Config.UseListR + fs.Config.UseListR = true + return func() { + fs.Config.UseListR = previous + } +} + // TestFsString tests the String method func TestFsString(t *testing.T) { skipIfNotOk(t) @@ -187,6 +201,12 @@ func TestFsListDirEmpty(t *testing.T) { assert.Equal(t, []string{}, dirsToNames(dirs)) } +// TestFsListRDirEmpty tests listing the directories from an empty directory using ListR +func TestFsListRDirEmpty(t *testing.T) { + defer skipIfNotListR(t)() + TestFsListDirEmpty(t) +} + // TestFsNewObjectNotFound tests not finding a object func TestFsNewObjectNotFound(t *testing.T) { skipIfNotOk(t) @@ -340,6 +360,13 @@ func TestFsListDirFile2(t *testing.T) { } } +// TestFsListRDirFile2 tests the files are correctly uploaded by doing +// Depth 1 directory listings using ListR +func TestFsListRDirFile2(t *testing.T) { + defer skipIfNotListR(t)() + TestFsListDirFile2(t) +} + // TestFsListDirRoot tests that DirList works in the root func TestFsListDirRoot(t *testing.T) { skipIfNotOk(t) @@ -350,6 +377,12 @@ func TestFsListDirRoot(t *testing.T) { assert.Contains(t, dirsToNames(dirs), subRemoteLeaf, "Remote leaf not found") } +// TestFsListRDirRoot tests that DirList works in the root using ListR +func TestFsListRDirRoot(t *testing.T) { + defer skipIfNotListR(t)() + TestFsListDirRoot(t) +} + // TestFsListSubdir tests List works for a subdirectory func TestFsListSubdir(t *testing.T) { skipIfNotOk(t) @@ -372,6 +405,12 @@ func TestFsListSubdir(t *testing.T) { require.Len(t, dirs, 0) } +// TestFsListRSubdir tests List works for a subdirectory using ListR +func TestFsListRSubdir(t *testing.T) { + defer skipIfNotListR(t)() + TestFsListSubdir(t) +} + // TestFsListLevel2 tests List works for 2 levels func TestFsListLevel2(t *testing.T) { skipIfNotOk(t) @@ -384,6 +423,12 @@ func TestFsListLevel2(t *testing.T) { assert.Equal(t, []string{`hello_ sausage`, `hello_ sausage/êé`}, dirsToNames(dirs)) } +// TestFsListRLevel2 tests List works for 2 levels using ListR +func TestFsListRLevel2(t *testing.T) { + defer skipIfNotListR(t)() + TestFsListLevel2(t) +} + // TestFsListFile1 tests file present func TestFsListFile1(t *testing.T) { skipIfNotOk(t) diff --git a/ftp/ftp.go b/ftp/ftp.go index c847b0555..cc661ab75 100644 --- a/ftp/ftp.go +++ b/ftp/ftp.go @@ -296,18 +296,25 @@ func (f *Fs) NewObject(remote string) (o fs.Object, err error) { return nil, fs.ErrorObjectNotFound } -func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) { +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { // defer fs.Trace(dir, "curlevel=%d", curlevel)("") c, err := f.getFtpConnection() if err != nil { - out.SetError(errors.Wrap(err, "list")) - return + return nil, errors.Wrap(err, "list") } files, err := c.List(path.Join(f.root, dir)) f.putFtpConnection(&c, err) if err != nil { - out.SetError(translateErrorDir(err)) - return + return nil, translateErrorDir(err) } for i := range files { object := files[i] @@ -317,20 +324,13 @@ func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) { if object.Name == "." || object.Name == ".." { continue } - if out.IncludeDirectory(newremote) { - d := &fs.Dir{ - Name: newremote, - When: object.Time, - Bytes: 0, - Count: -1, - } - if curlevel < out.Level() { - f.list(out, path.Join(dir, object.Name), curlevel+1) - } - if out.AddDir(d) { - return - } + d := &fs.Dir{ + Name: newremote, + When: object.Time, + Bytes: 0, + Count: -1, } + entries = append(entries, d) default: o := &Object{ fs: f, @@ -342,27 +342,10 @@ func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) { ModTime: object.Time, } o.info = info - if out.Add(o) { - return - } + entries = append(entries, o) } } -} - -// List the objects and directories of the Fs starting from dir -// -// dir should be "" to start from the root, and should not -// have trailing slashes. -// -// This should return ErrDirNotFound (using out.SetError()) -// if the directory isn't found. -// -// Fses must support recursion levels of fs.MaxLevel and 1. -// They may return ErrorLevelNotSupported otherwise. -func (f *Fs) List(out fs.ListOpts, dir string) { - // defer fs.Trace(dir, "")("") - f.list(out, dir, 1) - out.Finished() + return entries, nil } // Hashes are not supported diff --git a/ftp/ftp_test.go b/ftp/ftp_test.go index f3c944908..b3e62e023 100644 --- a/ftp/ftp_test.go +++ b/ftp/ftp_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/googlecloudstorage/googlecloudstorage.go b/googlecloudstorage/googlecloudstorage.go index 873eb645c..430b17a44 100644 --- a/googlecloudstorage/googlecloudstorage.go +++ b/googlecloudstorage/googlecloudstorage.go @@ -308,27 +308,28 @@ type listFn func(remote string, object *storage.Object, isDirectory bool) error // // dir is the starting directory, "" for root // -// If directories is set it only sends directories -func (f *Fs) list(dir string, level int, fn listFn) error { +// Set recurse to read sub directories +func (f *Fs) list(dir string, recurse bool, fn listFn) error { root := f.root rootLength := len(root) if dir != "" { root += dir + "/" } list := f.svc.Objects.List(f.bucket).Prefix(root).MaxResults(listChunks) - switch level { - case 1: + if !recurse { list = list.Delimiter("/") - case fs.MaxLevel: - default: - return fs.ErrorLevelNotSupported } for { objects, err := list.Do() if err != nil { + if gErr, ok := err.(*googleapi.Error); ok { + if gErr.Code == http.StatusNotFound { + err = fs.ErrorDirNotFound + } + } return err } - if level == 1 { + if !recurse { var object storage.Object for _, prefix := range objects.Prefixes { if !strings.HasSuffix(prefix, "/") { @@ -359,94 +360,120 @@ func (f *Fs) list(dir string, level int, fn listFn) error { return nil } -// listFiles lists files and directories to out -func (f *Fs) listFiles(out fs.ListOpts, dir string) { - defer out.Finished() - if f.bucket == "" { - out.SetError(errors.New("can't list objects at root - choose a bucket using lsd")) - return +// Convert a list item into a BasicInfo +func (f *Fs) itemToDirEntry(remote string, object *storage.Object, isDirectory bool) (fs.BasicInfo, error) { + if isDirectory { + d := &fs.Dir{ + Name: remote, + Bytes: int64(object.Size), + Count: 0, + } + return d, nil } + o, err := f.newObjectWithInfo(remote, object) + if err != nil { + return nil, err + } + return o, nil +} + +// listDir lists a single directory +func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { // List the objects - err := f.list(dir, out.Level(), func(remote string, object *storage.Object, isDirectory bool) error { - if isDirectory { - dir := &fs.Dir{ - Name: remote, - Bytes: int64(object.Size), - Count: 0, - } - if out.AddDir(dir) { - return fs.ErrorListAborted - } - } else { - o, err := f.newObjectWithInfo(remote, object) - if err != nil { - return err - } - if out.Add(o) { - return fs.ErrorListAborted - } + err = f.list(dir, false, func(remote string, object *storage.Object, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + if entry != nil { + entries = append(entries, entry) } return nil }) if err != nil { - if gErr, ok := err.(*googleapi.Error); ok { - if gErr.Code == http.StatusNotFound { - err = fs.ErrorDirNotFound - } - } - out.SetError(err) + return nil, err } + return entries, err } -// listBuckets lists the buckets to out -func (f *Fs) listBuckets(out fs.ListOpts, dir string) { - defer out.Finished() +// listBuckets lists the buckets +func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) { if dir != "" { - out.SetError(fs.ErrorListOnlyRoot) - return + return nil, fs.ErrorListBucketRequired } if f.projectNumber == "" { - out.SetError(errors.New("can't list buckets without project number")) - return + return nil, errors.New("can't list buckets without project number") } listBuckets := f.svc.Buckets.List(f.projectNumber).MaxResults(listChunks) for { buckets, err := listBuckets.Do() if err != nil { - out.SetError(err) - return + return nil, err } for _, bucket := range buckets.Items { - dir := &fs.Dir{ + d := &fs.Dir{ Name: bucket.Name, Bytes: 0, Count: 0, } - if out.AddDir(dir) { - return - } + entries = append(entries, d) } if buckets.NextPageToken == "" { break } listBuckets.PageToken(buckets.NextPageToken) } + return entries, nil } -// List lists the path to out -func (f *Fs) List(out fs.ListOpts, dir string) { +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { if f.bucket == "" { - f.listBuckets(out, dir) - } else { - f.listFiles(out, dir) + return f.listBuckets(dir) } - return + return f.listDir(dir) } // ListR lists the objects and directories of the Fs starting // from dir recursively into out. -func (f *Fs) ListR(out fs.ListOpts, dir string) { - f.List(out, dir) // FIXME +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + if f.bucket == "" { + return fs.ErrorListBucketRequired + } + list := fs.NewListRHelper(callback) + err = f.list(dir, true, func(remote string, object *storage.Object, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + return list.Add(entry) + }) + if err != nil { + return err + } + return list.Flush() } // Put the object into the bucket diff --git a/googlecloudstorage/googlecloudstorage_test.go b/googlecloudstorage/googlecloudstorage_test.go index 0c2c5cfad..57e18b062 100644 --- a/googlecloudstorage/googlecloudstorage_test.go +++ b/googlecloudstorage/googlecloudstorage_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/hubic/hubic_test.go b/hubic/hubic_test.go index d29fe423c..a20e76920 100644 --- a/hubic/hubic_test.go +++ b/hubic/hubic_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/local/local.go b/local/local.go index 252e95135..7536a8c42 100644 --- a/local/local.go +++ b/local/local.go @@ -181,26 +181,32 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) { return f.newObjectWithInfo(remote, "", nil) } -// listArgs is the arguments that a new list takes -type listArgs struct { - remote string - dirpath string - level int -} - -// list traverses the directory passed in, listing to out. -// it returns a boolean whether it is finished or not. -func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (subdirs []listArgs) { - fd, err := os.Open(dirpath) +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + dir = f.dirNames.Load(dir) + fsDirPath := f.cleanPath(filepath.Join(f.root, dir)) + remote := f.cleanRemote(dir) + _, err = os.Stat(fsDirPath) if err != nil { - out.SetError(errors.Wrapf(err, "failed to open directory %q", dirpath)) - return nil + return nil, fs.ErrorDirNotFound } + fd, err := os.Open(fsDirPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to open directory %q", dir) + } defer func() { - err := fd.Close() - if err != nil { - out.SetError(errors.Wrapf(err, "failed to close directory %q:", dirpath)) + cerr := fd.Close() + if cerr != nil && err == nil { + err = errors.Wrapf(cerr, "failed to close directory %q:", dir) } }() @@ -210,106 +216,46 @@ func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (su break } if err != nil { - out.SetError(errors.Wrapf(err, "failed to read directory %q", dirpath)) - - return nil + return nil, errors.Wrapf(err, "failed to read directory %q", dir) } for _, fi := range fis { name := fi.Name() mode := fi.Mode() newRemote := path.Join(remote, name) - newPath := filepath.Join(dirpath, name) + newPath := filepath.Join(fsDirPath, name) // Follow symlinks if required if *followSymlinks && (mode&os.ModeSymlink) != 0 { fi, err = os.Stat(newPath) if err != nil { - out.SetError(err) - return nil + return nil, err } mode = fi.Mode() } if fi.IsDir() { // Ignore directories which are symlinks. These are junction points under windows which // are kind of a souped up symlink. Unix doesn't have directories which are symlinks. - if (mode&os.ModeSymlink) == 0 && out.IncludeDirectory(newRemote) && f.dev == readDevice(fi) { - dir := &fs.Dir{ + if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi) { + d := &fs.Dir{ Name: f.dirNames.Save(newRemote, f.cleanRemote(newRemote)), When: fi.ModTime(), Bytes: 0, Count: 0, } - if out.AddDir(dir) { - return nil - } - if level > 0 { - subdirs = append(subdirs, listArgs{remote: newRemote, dirpath: newPath, level: level - 1}) - } + entries = append(entries, d) } } else { fso, err := f.newObjectWithInfo(newRemote, newPath, fi) if err != nil { - out.SetError(err) - return nil + return nil, err } - if fso.Storable() && out.Add(fso) { - return nil + if fso.Storable() { + entries = append(entries, fso) } } } } - return subdirs -} - -// List the path into out -// -// Ignores everything which isn't Storable, eg links etc -func (f *Fs) List(out fs.ListOpts, dir string) { - defer out.Finished() - dir = f.dirNames.Load(dir) - root := f.cleanPath(filepath.Join(f.root, dir)) - dir = f.cleanRemote(dir) - _, err := os.Stat(root) - if err != nil { - out.SetError(fs.ErrorDirNotFound) - return - } - - in := make(chan listArgs, out.Buffer()) - var wg sync.WaitGroup // sync closing of go routines - var traversing sync.WaitGroup // running directory traversals - - // Start the process - traversing.Add(1) - in <- listArgs{remote: dir, dirpath: root, level: out.Level() - 1} - for i := 0; i < fs.Config.Checkers; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for job := range in { - if out.IsFinished() { - continue - } - newJobs := f.list(out, job.remote, job.dirpath, job.level) - // Now we have traversed this directory, send - // these ones off for traversal - if len(newJobs) != 0 { - traversing.Add(len(newJobs)) - go func() { - for _, newJob := range newJobs { - in <- newJob - } - }() - } - traversing.Done() - } - }() - } - - // Wait for traversal to finish - traversing.Wait() - close(in) - wg.Wait() + return entries, nil } // cleanRemote makes string a valid UTF-8 string for remote strings. diff --git a/local/local_test.go b/local/local_test.go index b3b3025f7..b14acb473 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/onedrive/onedrive.go b/onedrive/onedrive.go index 208f9804e..edf6da438 100644 --- a/onedrive/onedrive.go +++ b/onedrive/onedrive.go @@ -399,50 +399,57 @@ OUTER: return } -// ListDir reads the directory specified by the job into out, returning any more jobs -func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.ListDirJob, err error) { - fs.Debugf(f, "Reading %q", job.Path) - _, err = f.listAll(job.DirID, false, false, func(info *api.Item) bool { - remote := job.Path + info.Name +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + err = f.dirCache.FindRoot(false) + if err != nil { + return nil, err + } + directoryID, err := f.dirCache.FindDir(dir, false) + if err != nil { + return nil, err + } + var iErr error + _, err = f.listAll(directoryID, false, false, func(info *api.Item) bool { + remote := path.Join(dir, info.Name) if info.Folder != nil { - if out.IncludeDirectory(remote) { - // cache the directory ID for later lookups - f.dirCache.Put(remote, info.ID) - dir := &fs.Dir{ - Name: remote, - Bytes: -1, - Count: -1, - When: time.Time(info.LastModifiedDateTime), - } - if info.Folder != nil { - dir.Count = info.Folder.ChildCount - } - if out.AddDir(dir) { - return true - } - if job.Depth > 0 { - jobs = append(jobs, dircache.ListDirJob{DirID: info.ID, Path: remote + "/", Depth: job.Depth - 1}) - } + // cache the directory ID for later lookups + f.dirCache.Put(remote, info.ID) + d := &fs.Dir{ + Name: remote, + Bytes: -1, + Count: -1, + When: time.Time(info.LastModifiedDateTime), } + if info.Folder != nil { + d.Count = info.Folder.ChildCount + } + entries = append(entries, d) } else { o, err := f.newObjectWithInfo(remote, info) if err != nil { - out.SetError(err) - return true - } - if out.Add(o) { + iErr = err return true } + entries = append(entries, o) } return false }) - fs.Debugf(f, "Finished reading %q", job.Path) - return jobs, err -} - -// List walks the path returning files and directories into out -func (f *Fs) List(out fs.ListOpts, dir string) { - f.dirCache.List(f, out, dir) + if err != nil { + return nil, err + } + if iErr != nil { + return nil, iErr + } + return entries, nil } // Creates from the parameters passed in a half finished Object which diff --git a/onedrive/onedrive_test.go b/onedrive/onedrive_test.go index 1a065ee53..ff1e504ab 100644 --- a/onedrive/onedrive_test.go +++ b/onedrive/onedrive_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/s3/s3.go b/s3/s3.go index 0af186cdb..63649e3f0 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -470,20 +470,16 @@ type listFn func(remote string, object *s3.Object, isDirectory bool) error // // dir is the starting directory, "" for root // -// Level is the level of the recursion -func (f *Fs) list(dir string, level int, fn listFn) error { +// Set recurse to read sub directories +func (f *Fs) list(dir string, recurse bool, fn listFn) error { root := f.root if dir != "" { root += dir + "/" } maxKeys := int64(listChunkSize) delimiter := "" - switch level { - case 1: + if !recurse { delimiter = "/" - case fs.MaxLevel: - default: - return fs.ErrorLevelNotSupported } var marker *string for { @@ -497,10 +493,15 @@ func (f *Fs) list(dir string, level int, fn listFn) error { } resp, err := f.c.ListObjects(&req) if err != nil { + if awsErr, ok := err.(awserr.RequestFailure); ok { + if awsErr.StatusCode() == http.StatusNotFound { + err = fs.ErrorDirNotFound + } + } return err } rootLength := len(f.root) - if level == 1 { + if !recurse { for _, commonPrefix := range resp.CommonPrefixes { if commonPrefix.Prefix == nil { fs.Logf(f, "Nil common prefix received") @@ -546,90 +547,116 @@ func (f *Fs) list(dir string, level int, fn listFn) error { return nil } -// listFiles lists files and directories to out -func (f *Fs) listFiles(out fs.ListOpts, dir string) { - defer out.Finished() - if f.bucket == "" { - // Return no objects at top level list - out.SetError(errors.New("can't list objects at root - choose a bucket using lsd")) - return +// Convert a list item into a BasicInfo +func (f *Fs) itemToDirEntry(remote string, object *s3.Object, isDirectory bool) (fs.BasicInfo, error) { + if isDirectory { + size := int64(0) + if object.Size != nil { + size = *object.Size + } + d := &fs.Dir{ + Name: remote, + Bytes: size, + Count: 0, + } + return d, nil } + o, err := f.newObjectWithInfo(remote, object) + if err != nil { + return nil, err + } + return o, nil +} + +// listDir lists files and directories to out +func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { // List the objects and directories - err := f.list(dir, out.Level(), func(remote string, object *s3.Object, isDirectory bool) error { - if isDirectory { - size := int64(0) - if object.Size != nil { - size = *object.Size - } - dir := &fs.Dir{ - Name: remote, - Bytes: size, - Count: 0, - } - if out.AddDir(dir) { - return fs.ErrorListAborted - } - } else { - o, err := f.newObjectWithInfo(remote, object) - if err != nil { - return err - } - if out.Add(o) { - return fs.ErrorListAborted - } + err = f.list(dir, false, func(remote string, object *s3.Object, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + if entry != nil { + entries = append(entries, entry) } return nil }) if err != nil { - if awsErr, ok := err.(awserr.RequestFailure); ok { - if awsErr.StatusCode() == http.StatusNotFound { - err = fs.ErrorDirNotFound - } - } - out.SetError(err) + return nil, err } + return entries, nil } // listBuckets lists the buckets to out -func (f *Fs) listBuckets(out fs.ListOpts, dir string) { - defer out.Finished() +func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) { if dir != "" { - out.SetError(fs.ErrorListOnlyRoot) - return + return nil, fs.ErrorListBucketRequired } req := s3.ListBucketsInput{} resp, err := f.c.ListBuckets(&req) if err != nil { - out.SetError(err) - return + return nil, err } for _, bucket := range resp.Buckets { - dir := &fs.Dir{ + d := &fs.Dir{ Name: aws.StringValue(bucket.Name), When: aws.TimeValue(bucket.CreationDate), Bytes: -1, Count: -1, } - if out.AddDir(dir) { - break - } + entries = append(entries, d) } + return entries, nil } -// List lists files and directories to out -func (f *Fs) List(out fs.ListOpts, dir string) { +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { if f.bucket == "" { - f.listBuckets(out, dir) - } else { - f.listFiles(out, dir) + return f.listBuckets(dir) } - return + return f.listDir(dir) } // ListR lists the objects and directories of the Fs starting // from dir recursively into out. -func (f *Fs) ListR(out fs.ListOpts, dir string) { - f.List(out, dir) // FIXME +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + if f.bucket == "" { + return fs.ErrorListBucketRequired + } + list := fs.NewListRHelper(callback) + err = f.list(dir, true, func(remote string, object *s3.Object, isDirectory bool) error { + entry, err := f.itemToDirEntry(remote, object, isDirectory) + if err != nil { + return err + } + return list.Add(entry) + }) + if err != nil { + return err + } + return list.Flush() } // Put the Object into the bucket diff --git a/s3/s3_test.go b/s3/s3_test.go index 40a875a1a..45ea9d6d8 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/sftp/sftp.go b/sftp/sftp.go index 262113708..77df26365 100644 --- a/sftp/sftp.go +++ b/sftp/sftp.go @@ -8,7 +8,6 @@ import ( "io" "os" "path" - "sync" "time" "github.com/ncw/rclone/fs" @@ -219,74 +218,52 @@ func (f *Fs) dirExists(dir string) (bool, error) { return true, nil } -func (f *Fs) list(out fs.ListOpts, dir string, level int, wg *sync.WaitGroup, tokens chan struct{}) { - defer wg.Done() - // take a token - <-tokens - // return it when done - defer func() { - tokens <- struct{}{} - }() - sftpDir := path.Join(f.root, dir) +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { + root := path.Join(f.root, dir) + ok, err := f.dirExists(root) + if err != nil { + return nil, errors.Wrap(err, "List failed") + } + if !ok { + return nil, fs.ErrorDirNotFound + } + sftpDir := root if sftpDir == "" { sftpDir = "." } infos, err := f.sftpClient.ReadDir(sftpDir) if err != nil { - err = errors.Wrapf(err, "error listing %q", dir) - fs.Errorf(f, "Listing failed: %v", err) - out.SetError(err) - return + return nil, errors.Wrapf(err, "error listing %q", dir) } for _, info := range infos { remote := path.Join(dir, info.Name()) if info.IsDir() { - if out.IncludeDirectory(remote) { - dir := &fs.Dir{ - Name: remote, - When: info.ModTime(), - Bytes: -1, - Count: -1, - } - out.AddDir(dir) - if level < out.Level() { - wg.Add(1) - go f.list(out, remote, level+1, wg, tokens) - } + d := &fs.Dir{ + Name: remote, + When: info.ModTime(), + Bytes: -1, + Count: -1, } + entries = append(entries, d) } else { - file := &Object{ + o := &Object{ fs: f, remote: remote, info: info, } - out.Add(file) + entries = append(entries, o) } } -} - -// List the files and directories starting at -func (f *Fs) List(out fs.ListOpts, dir string) { - root := path.Join(f.root, dir) - ok, err := f.dirExists(root) - if err != nil { - out.SetError(errors.Wrap(err, "List failed")) - return - } - if !ok { - out.SetError(fs.ErrorDirNotFound) - return - } - // tokens to control the concurrency - tokens := make(chan struct{}, fs.Config.Checkers) - for i := 0; i < fs.Config.Checkers; i++ { - tokens <- struct{}{} - } - wg := new(sync.WaitGroup) - wg.Add(1) - f.list(out, dir, 1, wg, tokens) - wg.Wait() - out.Finished() + return entries, nil } // Put data from into a new remote sftp file object described by and diff --git a/sftp/sftp_test.go b/sftp/sftp_test.go index d2cf9728f..8e81a6b75 100644 --- a/sftp/sftp_test.go +++ b/sftp/sftp_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/swift/swift.go b/swift/swift.go index 1e577b03a..9fdad6514 100644 --- a/swift/swift.go +++ b/swift/swift.go @@ -273,8 +273,8 @@ type listFn func(remote string, object *swift.Object, isDirectory bool) error // listContainerRoot lists the objects into the function supplied from // the container and root supplied // -// Level is the level of the recursion -func (f *Fs) listContainerRoot(container, root string, dir string, level int, fn listFn) error { +// Set recurse to read sub directories +func (f *Fs) listContainerRoot(container, root string, dir string, recurse bool, fn listFn) error { prefix := root if dir != "" { prefix += dir + "/" @@ -284,12 +284,8 @@ func (f *Fs) listContainerRoot(container, root string, dir string, level int, fn Prefix: prefix, Limit: listChunks, } - switch level { - case 1: + if !recurse { opts.Delimiter = '/' - case fs.MaxLevel: - default: - return fs.ErrorLevelNotSupported } rootLength := len(root) return f.c.ObjectsWalk(container, &opts, func(opts *swift.ObjectsOpts) (interface{}, error) { @@ -298,7 +294,7 @@ func (f *Fs) listContainerRoot(container, root string, dir string, level int, fn for i := range objects { object := &objects[i] isDirectory := false - if level == 1 { + if !recurse { if strings.HasSuffix(object.Name, "/") { isDirectory = true object.Name = object.Name[:len(object.Name)-1] @@ -319,29 +315,18 @@ func (f *Fs) listContainerRoot(container, root string, dir string, level int, fn }) } -// list the objects into the function supplied -func (f *Fs) list(dir string, level int, fn listFn) error { - return f.listContainerRoot(f.container, f.root, dir, level, fn) -} +type addEntryFn func(fs.BasicInfo) error -// listFiles walks the path returning a channel of Objects -func (f *Fs) listFiles(out fs.ListOpts, dir string) { - defer out.Finished() - if f.container == "" { - out.SetError(errors.New("can't list objects at root - choose a container using lsd")) - return - } - // List the objects - err := f.list(dir, out.Level(), func(remote string, object *swift.Object, isDirectory bool) error { +// list the objects into the function supplied +func (f *Fs) list(dir string, recurse bool, fn addEntryFn) error { + return f.listContainerRoot(f.container, f.root, dir, recurse, func(remote string, object *swift.Object, isDirectory bool) (err error) { if isDirectory { - dir := &fs.Dir{ + d := &fs.Dir{ Name: remote, Bytes: object.Bytes, Count: 0, } - if out.AddDir(dir) { - return fs.ErrorListAborted - } + err = fn(d) } else { o, err := f.newObjectWithInfo(remote, object) if err != nil { @@ -349,59 +334,96 @@ func (f *Fs) listFiles(out fs.ListOpts, dir string) { } // Storable does a full metadata read on 0 size objects which might be dynamic large objects if o.Storable() { - if out.Add(o) { - return fs.ErrorListAborted - } + err = fn(o) } } + return err + }) +} + +// listDir lists a single directory +func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) { + if f.container == "" { + return nil, fs.ErrorListBucketRequired + } + // List the objects + err = f.list(dir, false, func(entry fs.BasicInfo) error { + entries = append(entries, entry) return nil }) if err != nil { if err == swift.ContainerNotFound { err = fs.ErrorDirNotFound } - out.SetError(err) + return nil, err } + return entries, nil } // listContainers lists the containers -func (f *Fs) listContainers(out fs.ListOpts, dir string) { - defer out.Finished() +func (f *Fs) listContainers(dir string) (entries fs.DirEntries, err error) { if dir != "" { - out.SetError(fs.ErrorListOnlyRoot) - return + return nil, fs.ErrorListBucketRequired } containers, err := f.c.ContainersAll(nil) if err != nil { - out.SetError(err) - return + return nil, errors.Wrap(err, "container listing failed") } for _, container := range containers { - dir := &fs.Dir{ + d := &fs.Dir{ Name: container.Name, Bytes: container.Bytes, Count: container.Count, } - if out.AddDir(dir) { - break - } + entries = append(entries, d) } + return entries, nil } -// List walks the path returning files and directories to out -func (f *Fs) List(out fs.ListOpts, dir string) { +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { if f.container == "" { - f.listContainers(out, dir) - } else { - f.listFiles(out, dir) + return f.listContainers(dir) } - return + return f.listDir(dir) } // ListR lists the objects and directories of the Fs starting // from dir recursively into out. -func (f *Fs) ListR(out fs.ListOpts, dir string) { - f.List(out, dir) // FIXME +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { + if f.container == "" { + return errors.New("container needed for recursive list") + } + list := fs.NewListRHelper(callback) + err = f.list(dir, true, func(entry fs.BasicInfo) error { + return list.Add(entry) + }) + if err != nil { + return err + } + return list.Flush() } // Put the object into the container @@ -471,12 +493,8 @@ func (f *Fs) Purge() error { go func() { delErr <- fs.DeleteFiles(toBeDeleted) }() - err := f.list("", fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error { - if !isDirectory { - o, err := f.newObjectWithInfo(remote, object) - if err != nil { - return err - } + err := f.list("", true, func(entry fs.BasicInfo) error { + if o, ok := entry.(*Object); ok { toBeDeleted <- o } return nil @@ -679,7 +697,7 @@ func min(x, y int64) int64 { // if except is passed in then segments with that prefix won't be deleted func (o *Object) removeSegments(except string) error { segmentsRoot := o.fs.root + o.remote + "/" - err := o.fs.listContainerRoot(o.fs.segmentsContainer, segmentsRoot, "", fs.MaxLevel, func(remote string, object *swift.Object, isDirectory bool) error { + err := o.fs.listContainerRoot(o.fs.segmentsContainer, segmentsRoot, "", true, func(remote string, object *swift.Object, isDirectory bool) error { if isDirectory { return nil } diff --git a/swift/swift_test.go b/swift/swift_test.go index 0887fe389..1190a0f14 100644 --- a/swift/swift_test.go +++ b/swift/swift_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } diff --git a/yandex/yandex.go b/yandex/yandex.go index 5b16f134f..8e99d60ad 100644 --- a/yandex/yandex.go +++ b/yandex/yandex.go @@ -166,11 +166,43 @@ func (f *Fs) setRoot(root string) { f.diskRoot = diskRoot } -// listFn is called from list and listContainerRoot to handle an object. -type listFn func(remote string, item *yandex.ResourceInfoResponse, isDirectory bool) error +// Convert a list item into a BasicInfo +func (f *Fs) itemToDirEntry(remote string, object *yandex.ResourceInfoResponse) (fs.BasicInfo, error) { + switch object.ResourceType { + case "dir": + t, err := time.Parse(time.RFC3339Nano, object.Modified) + if err != nil { + return nil, errors.Wrap(err, "error parsing time in directory item") + } + d := &fs.Dir{ + Name: remote, + When: t, + Bytes: int64(object.Size), + Count: -1, + } + return d, nil + case "file": + o, err := f.newObjectWithInfo(remote, object) + if err != nil { + return nil, err + } + return o, nil + default: + fs.Debugf(f, "Unknown resource type %q", object.ResourceType) + } + return nil, nil +} -// listDir lists this directory only returning objects and directories -func (f *Fs) listDir(dir string, fn listFn) (err error) { +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { //request object meta info var opt yandex.ResourceInfoRequestOptions root := f.diskRoot @@ -189,30 +221,22 @@ func (f *Fs) listDir(dir string, fn listFn) (err error) { if err != nil { yErr, ok := err.(yandex.DiskClientError) if ok && yErr.Code == "DiskNotFoundError" { - return fs.ErrorDirNotFound + return nil, fs.ErrorDirNotFound } - return err + return nil, err } itemsCount = uint32(len(ResourceInfoResponse.Embedded.Items)) if ResourceInfoResponse.ResourceType == "dir" { //list all subdirs - for i, element := range ResourceInfoResponse.Embedded.Items { + for _, element := range ResourceInfoResponse.Embedded.Items { remote := path.Join(dir, element.Name) - fs.Debugf(i, "%q", remote) - switch element.ResourceType { - case "dir": - err = fn(remote, &element, true) - if err != nil { - return err - } - case "file": - err = fn(remote, &element, false) - if err != nil { - return err - } - default: - fs.Debugf(f, "Unknown resource type %q", element.ResourceType) + entry, err := f.itemToDirEntry(remote, &element) + if err != nil { + return nil, err + } + if entry != nil { + entries = append(entries, entry) } } } @@ -224,13 +248,26 @@ func (f *Fs) listDir(dir string, fn listFn) (err error) { break } } - return nil + return entries, nil } -// list the objects into the function supplied +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. // -// This does a flat listing of all the files in the drive -func (f *Fs) list(dir string, fn listFn) error { +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { //request files list. list is divided into pages. We send request for each page //items per page is limited by limit //TODO may be add config parameter for the items per page limit @@ -259,17 +296,26 @@ func (f *Fs) list(dir string, fn listFn) error { itemsCount = uint32(len(info.Items)) //list files + entries := make(fs.DirEntries, 0, len(info.Items)) for _, item := range info.Items { // filter file list and get only files we need if strings.HasPrefix(item.Path, prefix) { //trim root folder from filename var name = strings.TrimPrefix(item.Path, f.diskRoot) - err = fn(name, &item, false) + entry, err := f.itemToDirEntry(name, &item) if err != nil { return err } + if entry != nil { + entries = append(entries, entry) + } } } + // send the listing + err = callback(entries) + if err != nil { + return err + } //offset for the next page of items offset += itemsCount @@ -281,57 +327,6 @@ func (f *Fs) list(dir string, fn listFn) error { return nil } -// List walks the path returning a channel of Objects -func (f *Fs) List(out fs.ListOpts, dir string) { - defer out.Finished() - - listItem := func(remote string, object *yandex.ResourceInfoResponse, isDirectory bool) error { - if isDirectory { - t, err := time.Parse(time.RFC3339Nano, object.Modified) - if err != nil { - return err - } - dir := &fs.Dir{ - Name: remote, - When: t, - Bytes: int64(object.Size), - Count: -1, - } - if out.AddDir(dir) { - return fs.ErrorListAborted - } - } else { - o, err := f.newObjectWithInfo(remote, object) - if err != nil { - return err - } - if out.Add(o) { - return fs.ErrorListAborted - } - } - return nil - } - - var err error - switch out.Level() { - case 1: - err = f.listDir(dir, listItem) - case fs.MaxLevel: - err = f.list(dir, listItem) - default: - out.SetError(fs.ErrorLevelNotSupported) - } - if err != nil { - out.SetError(err) - } -} - -// ListR lists the objects and directories of the Fs starting -// from dir recursively into out. -func (f *Fs) ListR(out fs.ListOpts, dir string) { - f.List(out, dir) // FIXME -} - // NewObject finds the Object at remote. If it can't be found it // returns the error fs.ErrorObjectNotFound. func (f *Fs) NewObject(remote string) (fs.Object, error) { @@ -657,6 +652,7 @@ var ( _ fs.Purger = (*Fs)(nil) _ fs.ListRer = (*Fs)(nil) //_ fs.Copier = (*Fs)(nil) + _ fs.ListRer = (*Fs)(nil) _ fs.Object = (*Object)(nil) _ fs.MimeTyper = &Object{} ) diff --git a/yandex/yandex_test.go b/yandex/yandex_test.go index ce04ca3ad..683f48dd3 100644 --- a/yandex/yandex_test.go +++ b/yandex/yandex_test.go @@ -26,15 +26,20 @@ func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }