Read folders in separate goroutines.

As proposed in the FIXME, read folders in parallel.
This appears to fix "Next token is expired" on very big directories.
The only downside is that this doesn't abort at once if an error is found.
I added some logging, so there is some output for "-v".
This commit is contained in:
Klaus Post 2015-09-12 21:59:14 +02:00 committed by Nick Craig-Wood
parent fa87077211
commit a1a780e847

View File

@ -28,6 +28,7 @@ import (
"github.com/ncw/rclone/oauthutil" "github.com/ncw/rclone/oauthutil"
"github.com/ncw/rclone/pacer" "github.com/ncw/rclone/pacer"
"golang.org/x/oauth2" "golang.org/x/oauth2"
"sync"
) )
const ( const (
@ -353,16 +354,23 @@ OUTER:
func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error { func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error {
var subError error var subError error
// Make the API request // Make the API request
var wg sync.WaitGroup
_, err := f.listAll(dirId, "", false, false, func(node *acd.Node) bool { _, err := f.listAll(dirId, "", false, false, func(node *acd.Node) bool {
// Recurse on directories // Recurse on directories
// FIXME should do this in parallel
// use a wg to sync then collect error
switch *node.Kind { switch *node.Kind {
case folderKind: case folderKind:
subError = f.listDirRecursive(*node.Id, path+*node.Name+"/", out) wg.Add(1)
if subError != nil { folder := path + *node.Name + "/"
return true fs.Debug(f, "Reading %s", folder)
} go func() {
defer wg.Done()
err := f.listDirRecursive(*node.Id, folder, out)
if err != nil {
subError = err
fs.ErrorLog(f, "Error reading %s:%s", folder, err)
}
}()
return false
case fileKind: case fileKind:
if fs := f.newFsObjectWithInfo(path+*node.Name, node); fs != nil { if fs := f.newFsObjectWithInfo(path+*node.Name, node); fs != nil {
out <- fs out <- fs
@ -372,6 +380,8 @@ func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan)
} }
return false return false
}) })
wg.Wait()
fs.Debug(f, "Finished reading %s", path)
if err != nil { if err != nil {
return err return err
} }