mirror of
https://github.com/rclone/rclone.git
synced 2024-12-23 15:38:57 +01:00
Read folders in separate goroutines.
As proposed in the FIXME, read folders in parallel. This appears to fix "Next token is expired" on very big directories. The only downside is that this doesn't abort at once if an error is found. I added some logging, so there is some output for "-v".
This commit is contained in:
parent
fa87077211
commit
a1a780e847
@ -28,6 +28,7 @@ import (
|
||||
"github.com/ncw/rclone/oauthutil"
|
||||
"github.com/ncw/rclone/pacer"
|
||||
"golang.org/x/oauth2"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -353,16 +354,23 @@ OUTER:
|
||||
func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error {
|
||||
var subError error
|
||||
// Make the API request
|
||||
var wg sync.WaitGroup
|
||||
_, err := f.listAll(dirId, "", false, false, func(node *acd.Node) bool {
|
||||
// Recurse on directories
|
||||
// FIXME should do this in parallel
|
||||
// use a wg to sync then collect error
|
||||
switch *node.Kind {
|
||||
case folderKind:
|
||||
subError = f.listDirRecursive(*node.Id, path+*node.Name+"/", out)
|
||||
if subError != nil {
|
||||
return true
|
||||
}
|
||||
wg.Add(1)
|
||||
folder := path + *node.Name + "/"
|
||||
fs.Debug(f, "Reading %s", folder)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
err := f.listDirRecursive(*node.Id, folder, out)
|
||||
if err != nil {
|
||||
subError = err
|
||||
fs.ErrorLog(f, "Error reading %s:%s", folder, err)
|
||||
}
|
||||
}()
|
||||
return false
|
||||
case fileKind:
|
||||
if fs := f.newFsObjectWithInfo(path+*node.Name, node); fs != nil {
|
||||
out <- fs
|
||||
@ -372,6 +380,8 @@ func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan)
|
||||
}
|
||||
return false
|
||||
})
|
||||
wg.Wait()
|
||||
fs.Debug(f, "Finished reading %s", path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user