Refactor the List and ListDir interface

Gives more accurate error propagation, control of depth of recursion
and short circuit recursion where possible.

Most of the the heavy lifting is done in the "fs" package, making file
system implementations a bit simpler.

This commit contains some code originally by Klaus Post.

Fixes #316

fs/fs.go

@@ -5,9 +5,12 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"math"
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strings"
+	"sync"
 	"time"
 )
 
@@ -18,6 +21,8 @@ const (
 	// ModTimeNotSupported is a very large precision value to show
 	// mod time isn't supported on this Fs
 	ModTimeNotSupported = 100 * 365 * 24 * time.Hour
+	// MaxLevel is a sentinel representing an infinite depth for listings
+	MaxLevel = math.MaxInt32
 )
 
 // Globals
@@ -32,6 +37,9 @@ var (
 	ErrorCantDirMove          = fmt.Errorf("Can't move directory - incompatible remotes")
 	ErrorDirExists            = fmt.Errorf("Can't copy directory - destination already exists")
 	ErrorCantSetModTime       = fmt.Errorf("Can't set modified time")
+	ErrorDirNotFound          = fmt.Errorf("Directory not found")
+	ErrorLevelNotSupported    = fmt.Errorf("Level value not supported")
+	ErrorListAborted          = fmt.Errorf("List aborted")
 )
 
 // RegInfo provides information about a filesystem
@@ -90,11 +98,10 @@ func Register(info *RegInfo) {
 type Fs interface {
 	Info
 
-	// List the Fs into a channel
-	List() ObjectsChan
-
-	// ListDir lists the Fs directories/buckets/containers into a channel
-	ListDir() DirChan
+	// List the objects and directories of the Fs
+	//
+	// This should return ErrDirNotFound if the directory isn't found.
+	List(ListOpts)
 
 	// NewFsObject finds the Object at remote.  Returns nil if can't be found
 	NewFsObject(remote string) Object
@@ -239,6 +246,292 @@ type UnWrapper interface {
 // ObjectsChan is a channel of Objects
 type ObjectsChan chan Object
 
+// ListOpts describes the interface used for Fs.List operations
+type ListOpts interface {
+	// Add an object to the output.
+	// If the function returns true, the operation has been aborted.
+	// Multiple goroutines can safely add objects concurrently.
+	Add(obj Object) (abort bool)
+
+	// Add a directory to the output.
+	// If the function returns true, the operation has been aborted.
+	// Multiple goroutines can safely add objects concurrently.
+	AddDir(dir *Dir) (abort bool)
+
+	// IncludeDirectory returns whether this directory should be
+	// included in the listing (and recursed into or not).
+	IncludeDirectory(remote string) bool
+
+	// SetError will set an error state, and will cause the listing to
+	// be aborted.
+	// Multiple goroutines can set the error state concurrently,
+	// but only the first will be returned to the caller.
+	SetError(err error)
+
+	// Level returns the level it should recurse to.  Fses may
+	// ignore this in which case the listing will be less
+	// efficient.
+	Level() int
+
+	// Buffer returns the channel depth in use
+	Buffer() int
+
+	// Finished should be called when listing is finished
+	Finished()
+
+	// IsFinished returns whether Finished or SetError have been called
+	IsFinished() bool
+}
+
+// listerResult is returned by the lister methods
+type listerResult struct {
+	Obj Object
+	Dir *Dir
+	Err error
+}
+
+// Lister objects are used for controlling listing of Fs objects
+type Lister struct {
+	mu       sync.RWMutex
+	buffer   int
+	abort    bool
+	results  chan listerResult
+	finished sync.Once
+	level    int
+	filter   *Filter
+}
+
+// NewLister creates a Lister object.
+//
+// The default channel buffer size will be Config.Checkers unless
+// overridden with SetBuffer.  The default level will be infinite.
+func NewLister() *Lister {
+	o := &Lister{}
+	return o.SetLevel(-1).SetBuffer(Config.Checkers)
+}
+
+// Start starts a go routine listing the Fs passed in.  It returns the
+// same Lister that was passed in for convenience.
+func (o *Lister) Start(f Fs) *Lister {
+	o.results = make(chan listerResult, o.buffer)
+	go func() {
+		f.List(o)
+	}()
+	return o
+}
+
+// SetLevel sets the level to recurse to.  It returns same Lister that
+// was passed in for convenience.  If Level is < 0 then it sets it to
+// infinite.  Must be called before Start().
+func (o *Lister) SetLevel(level int) *Lister {
+	if level < 0 {
+		o.level = MaxLevel
+	} else {
+		o.level = level
+	}
+	return o
+}
+
+// SetFilter sets the Filter that is in use.  It defaults to no
+// filtering.  Must be called before Start().
+func (o *Lister) SetFilter(filter *Filter) *Lister {
+	o.filter = filter
+	return o
+}
+
+// Level gets the recursion level for this listing.
+//
+// Fses may ignore this, but should implement it for improved efficiency if possible.
+//
+// Level 1 means list just the contents of the directory
+//
+// Each returned item must have less than level `/`s in.
+func (o *Lister) Level() int {
+	return o.level
+}
+
+// SetBuffer sets the channel buffer size in use.  Must be called
+// before Start().
+func (o *Lister) SetBuffer(buffer int) *Lister {
+	if buffer < 1 {
+		buffer = 1
+	}
+	o.buffer = buffer
+	return o
+}
+
+// Buffer gets the channel buffer size in use
+func (o *Lister) Buffer() int {
+	return o.buffer
+}
+
+// Add an object to the output.
+// If the function returns true, the operation has been aborted.
+// Multiple goroutines can safely add objects concurrently.
+func (o *Lister) Add(obj Object) (abort bool) {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+	if o.abort {
+		return true
+	}
+	o.results <- listerResult{Obj: obj}
+	return false
+}
+
+// AddDir will a directory to the output.
+// If the function returns true, the operation has been aborted.
+// Multiple goroutines can safely add objects concurrently.
+func (o *Lister) AddDir(dir *Dir) (abort bool) {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+	if o.abort {
+		return true
+	}
+	remote := dir.Name
+	remote = strings.Trim(remote, "/")
+	dir.Name = remote
+	// Check the level and ignore if too high
+	slashes := strings.Count(remote, "/")
+	if slashes >= o.level {
+		return false
+	}
+	// Check if directory is included
+	if !o.IncludeDirectory(remote) {
+		return false
+	}
+	o.results <- listerResult{Dir: dir}
+	return false
+}
+
+// IncludeDirectory returns whether this directory should be
+// included in the listing (and recursed into or not).
+func (o *Lister) IncludeDirectory(remote string) bool {
+	if o.filter == nil {
+		return true
+	}
+	return o.filter.IncludeDirectory(remote)
+}
+
+// SetError will set an error state, and will cause the listing to
+// be aborted.
+// Multiple goroutines can set the error state concurrently,
+// but only the first will be returned to the caller.
+func (o *Lister) SetError(err error) {
+	o.mu.RLock()
+	if err != nil && !o.abort {
+		o.results <- listerResult{Err: err}
+	}
+	o.mu.RUnlock()
+	o.Finished()
+}
+
+// Finished should be called when listing is finished
+func (o *Lister) Finished() {
+	o.finished.Do(func() {
+		o.mu.Lock()
+		o.abort = true
+		close(o.results)
+		o.mu.Unlock()
+	})
+}
+
+// IsFinished returns whether the directory listing is finished or not
+func (o *Lister) IsFinished() bool {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+	return o.abort
+}
+
+// Get an object from the listing.
+// Will return either an object or a directory, never both.
+// Will return (nil, nil, nil) when all objects have been returned.
+func (o *Lister) Get() (Object, *Dir, error) {
+	select {
+	case r := <-o.results:
+		return r.Obj, r.Dir, r.Err
+	}
+}
+
+// GetObject will return an object from the listing.
+// It will skip over any directories.
+// Will return (nil, nil) when all objects have been returned.
+func (o *Lister) GetObject() (Object, error) {
+	for {
+		obj, dir, err := o.Get()
+		if err != nil {
+			return nil, err
+		}
+		// Check if we are finished
+		if dir == nil && obj == nil {
+			return nil, nil
+		}
+		// Ignore directories
+		if dir != nil {
+			continue
+		}
+		return obj, nil
+	}
+}
+
+// GetObjects will return a slice of object from the listing.
+// It will skip over any directories.
+func (o *Lister) GetObjects() (objs []Object, err error) {
+	for {
+		obj, dir, err := o.Get()
+		if err != nil {
+			return nil, err
+		}
+		// Check if we are finished
+		if dir == nil && obj == nil {
+			break
+		}
+		if obj != nil {
+			objs = append(objs, obj)
+		}
+	}
+	return objs, nil
+}
+
+// GetDir will return a directory from the listing.
+// It will skip over any objects.
+// Will return (nil, nil) when all objects have been returned.
+func (o *Lister) GetDir() (*Dir, error) {
+	for {
+		obj, dir, err := o.Get()
+		if err != nil {
+			return nil, err
+		}
+		// Check if we are finished
+		if dir == nil && obj == nil {
+			return nil, nil
+		}
+		// Ignore objects
+		if obj != nil {
+			continue
+		}
+		return dir, nil
+	}
+}
+
+// GetDirs will return a slice of directories from the listing.
+// It will skip over any objects.
+func (o *Lister) GetDirs() (dirs []*Dir, err error) {
+	for {
+		obj, dir, err := o.Get()
+		if err != nil {
+			return nil, err
+		}
+		// Check if we are finished
+		if dir == nil && obj == nil {
+			break
+		}
+		if dir != nil {
+			dirs = append(dirs, dir)
+		}
+	}
+	return dirs, nil
+}
+
 // Objects is a slice of Object~s
 type Objects []Object