feat: local: list objects in parallel controlled by the --checkers option -- fixes #6632

Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>

backend/local/local.go

@@ -28,6 +28,7 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/file"
 	"github.com/rclone/rclone/lib/readers"
+	"golang.org/x/sync/errgroup"
 	"golang.org/x/text/unicode/norm"
 )
 
@@ -486,6 +487,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 	for {
 		var fis []os.FileInfo
+
 		if useReadDir {
 			// Windows and Plan9 read the directory entries with the stat information in which
 			// shouldn't fail because of unreadable entries.
@@ -495,71 +497,109 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			}
 		} else {
 			// For other OSes we read the names only (which shouldn't fail) then stat the
-			// individual ourselves so we can log errors but not fail the directory read.
+			// individual ourselves, so we can log errors but not fail the directory read.
 			var names []string
 			names, err = fd.Readdirnames(1024)
 			if err == io.EOF && len(names) == 0 {
 				break
 			}
+
+			fis = make([]os.FileInfo, len(names))
+
+			g, gCtx := errgroup.WithContext(ctx)
+			g.SetLimit(fs.GetConfig(ctx).Checkers)
 			if err == nil {
-				for _, name := range names {
+				for i, name := range names {
+					i, name := i, name // https://golang.org/doc/faq#closures_and_goroutines
+					g.Go(func() error {
+						// No point in continuing if context has been cancelled
+						if gCtx.Err() != nil {
+							return nil
+						}
+
+						var err error
 						namepath := filepath.Join(fsDirPath, name)
 						fi, fierr := os.Lstat(namepath)
 						if os.IsNotExist(fierr) {
 							// skip entry removed by a concurrent goroutine
-						continue
+							return nil
 						}
 						if fierr != nil {
-						// Don't report errors on any file names that are excluded
 							if useFilter {
 								newRemote := f.cleanRemote(dir, name)
 								if !filter.IncludeRemote(newRemote) {
-								continue
+									return nil
 								}
 							}
-						fierr = fmt.Errorf("failed to get info about directory entry %q: %w", namepath, fierr)
-						fs.Errorf(dir, "%v", fierr)
-						_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
-						continue
+							err = fmt.Errorf("failed to get info about directory entry %q: %w", namepath, fierr)
+							fs.Errorf(dir, "%v", err)
+							_ = accounting.Stats(gCtx).Error(fserrors.NoRetryError(err)) // fail the sync
+							return nil
 						}
-					fis = append(fis, fi)
+						fis[i] = fi
+						return nil
+					})
 				}
 			}
+			err = g.Wait()
 		}
 		if err != nil {
 			return nil, fmt.Errorf("failed to read directory entry: %w", err)
 		}
 
-		for _, fi := range fis {
+		loopEntries := make(fs.DirEntries, len(fis))
+
+		g, gCtx := errgroup.WithContext(ctx)
+		g.SetLimit(fs.GetConfig(ctx).Checkers)
+		for i, fi := range fis {
+			if fi == nil {
+				continue
+			}
+			i, fi := i, fi // https://golang.org/doc/faq#closures_and_goroutines
+			g.Go(func() error {
+				// No point in continuing if context has been cancelled
+				if gCtx.Err() != nil {
+					return nil
+				}
+
+				var err error
 				name := fi.Name()
 				mode := fi.Mode()
 				newRemote := f.cleanRemote(dir, name)
+
 				// Follow symlinks if required
 				if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
 					localPath := filepath.Join(fsDirPath, name)
 					fi, err = os.Stat(localPath)
+					if err != nil {
 						// Quietly skip errors on excluded files and directories
-				if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
-					continue
+						if useFilter && !filter.IncludeRemote(newRemote) {
+							return nil
 						}
 						if os.IsNotExist(err) || isCircularSymlinkError(err) {
 							// Skip bad symlinks and circular symlinks
 							err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
 							fs.Errorf(newRemote, "Listing error: %v", err)
-					err = accounting.Stats(ctx).Error(err)
-					continue
+							_ = accounting.Stats(gCtx).Error(err)
+							return nil
 						}
-				if err != nil {
-					return nil, err
+						return err
 					}
 					mode = fi.Mode()
 				}
+
+				// No point in continuing if context has been cancelled
+				if gCtx.Err() != nil {
+					return nil
+				}
+
 				if fi.IsDir() {
 					// Ignore directories which are symlinks.  These are junction points under windows which
 					// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
 					if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi, f.opt.OneFileSystem) {
 						d := fs.NewDir(newRemote, fi.ModTime())
-					entries = append(entries, d)
+						loopEntries[i] = d
+						return nil
 					}
 				} else {
 					// Check whether this link should be translated
@@ -569,18 +609,33 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					// Don't include non directory if not included
 					// we leave directory filtering to the layer above
 					if useFilter && !filter.IncludeRemote(newRemote) {
-					continue
+						return nil
 					}
 					fso, err := f.newObjectWithInfo(newRemote, fi)
 					if err != nil {
-					return nil, err
+						return err
 					}
 					if fso.Storable() {
-					entries = append(entries, fso)
+						loopEntries[i] = fso
+						return nil
 					}
 				}
+				return nil
+			})
+		}
+		err = g.Wait()
+		if err != nil {
+			return nil, err
+		}
+
+		for _, entry := range loopEntries {
+			if entry == nil {
+				continue
+			}
+			entries = append(entries, entry)
 		}
 	}
+
 	return entries, nil
 }