mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 16:34:30 +01:00
Read folders in separate goroutines.
As proposed in the FIXME, read folders in parallel. This appears to fix "Next token is expired" on very big directories. The only downside is that this doesn't abort at once if an error is found. I added some logging, so there is some output for "-v".
This commit is contained in:
parent
fa87077211
commit
a1a780e847
@ -28,6 +28,7 @@ import (
|
|||||||
"github.com/ncw/rclone/oauthutil"
|
"github.com/ncw/rclone/oauthutil"
|
||||||
"github.com/ncw/rclone/pacer"
|
"github.com/ncw/rclone/pacer"
|
||||||
"golang.org/x/oauth2"
|
"golang.org/x/oauth2"
|
||||||
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -353,16 +354,23 @@ OUTER:
|
|||||||
func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error {
|
func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan) error {
|
||||||
var subError error
|
var subError error
|
||||||
// Make the API request
|
// Make the API request
|
||||||
|
var wg sync.WaitGroup
|
||||||
_, err := f.listAll(dirId, "", false, false, func(node *acd.Node) bool {
|
_, err := f.listAll(dirId, "", false, false, func(node *acd.Node) bool {
|
||||||
// Recurse on directories
|
// Recurse on directories
|
||||||
// FIXME should do this in parallel
|
|
||||||
// use a wg to sync then collect error
|
|
||||||
switch *node.Kind {
|
switch *node.Kind {
|
||||||
case folderKind:
|
case folderKind:
|
||||||
subError = f.listDirRecursive(*node.Id, path+*node.Name+"/", out)
|
wg.Add(1)
|
||||||
if subError != nil {
|
folder := path + *node.Name + "/"
|
||||||
return true
|
fs.Debug(f, "Reading %s", folder)
|
||||||
}
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
err := f.listDirRecursive(*node.Id, folder, out)
|
||||||
|
if err != nil {
|
||||||
|
subError = err
|
||||||
|
fs.ErrorLog(f, "Error reading %s:%s", folder, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return false
|
||||||
case fileKind:
|
case fileKind:
|
||||||
if fs := f.newFsObjectWithInfo(path+*node.Name, node); fs != nil {
|
if fs := f.newFsObjectWithInfo(path+*node.Name, node); fs != nil {
|
||||||
out <- fs
|
out <- fs
|
||||||
@ -372,6 +380,8 @@ func (f *FsAcd) listDirRecursive(dirId string, path string, out fs.ObjectsChan)
|
|||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
})
|
})
|
||||||
|
wg.Wait()
|
||||||
|
fs.Debug(f, "Finished reading %s", path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user