Read folders in separate goroutines.

As proposed in the FIXME, read folders in parallel.
This appears to fix "Next token is expired" on very big directories.
The only downside is that this doesn't abort at once if an error is found.
I added some logging, so there is some output for "-v".
This commit is contained in:
Klaus Post 2015-09-12 21:59:14 +02:00 committed by Nick Craig-Wood
parent fa87077211
commit a1a780e847

View File

@ -28,6 +28,7 @@ import (
"github.com/ncw/rclone/oauthutil"
"github.com/ncw/rclone/pacer"
"golang.org/x/oauth2"
"sync"
)
const (
@ -353,16 +354,23 @@ OUTER:
func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error {
var subError error
// Make the API request
var wg sync.WaitGroup
_, err := f.listAll(dirId, "", false, false, func(node *acd.Node) bool {
// Recurse on directories
// FIXME should do this in parallel
// use a wg to sync then collect error
switch *node.Kind {
case folderKind:
subError = f.listDirRecursive(*node.Id, path+*node.Name+"/", out)
if subError != nil {
return true
wg.Add(1)
folder := path + *node.Name + "/"
fs.Debug(f, "Reading %s", folder)
go func() {
defer wg.Done()
err := f.listDirRecursive(*node.Id, folder, out)
if err != nil {
subError = err
fs.ErrorLog(f, "Error reading %s:%s", folder, err)
}
}()
return false
case fileKind:
if fs := f.newFsObjectWithInfo(path+*node.Name, node); fs != nil {
out <- fs
@ -372,6 +380,8 @@ func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan)
}
return false
})
wg.Wait()
fs.Debug(f, "Finished reading %s", path)
if err != nil {
return err
}