walk: do filtering in parallel to speed up rclone delete - FIXME okish

See: https://forum.rclone.org/t/rclone-delete-on-s3-very-slow/37543

fs/walk/walk.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rclone/rclone/fs/dirtree"
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/list"
+	"golang.org/x/sync/errgroup"
 )
 
 // ErrorSkipDir is used as a return value from Walk to indicate that the
@@ -294,20 +295,10 @@ func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType
 	if synthesizeDirs {
 		dm = newDirMap(path)
 	}
-	var mu sync.Mutex
-	err := doListR(ctx, path, func(entries fs.DirEntries) (err error) {
-		if synthesizeDirs {
-			err = dm.addEntries(entries)
-			if err != nil {
-				return err
-			}
-		}
-		listType.Filter(&entries)
-		if !includeAll {
-			filteredEntries := entries[:0]
-			for _, entry := range entries {
+	// nil the entry out if not included
+	filterEntry := func(ctx context.Context, entry *fs.DirEntry) (err error) {
 		var include bool
-				switch x := entry.(type) {
+		switch x := (*entry).(type) {
 		case fs.Object:
 			include = fi.IncludeObject(ctx, x)
 		case fs.Directory:
@@ -318,10 +309,39 @@ func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType
 		default:
 			return fmt.Errorf("unknown object type %T", entry)
 		}
-				if include {
-					filteredEntries = append(filteredEntries, entry)
-				} else {
+		if !include {
 			fs.Debugf(entry, "Excluded from sync (and deletion)")
+			*entry = nil
+		}
+		return nil
+	}
+	ci := fs.GetConfig(ctx)
+	var mu sync.Mutex
+	err := doListR(ctx, path, func(entries fs.DirEntries) (err error) {
+		if synthesizeDirs {
+			err = dm.addEntries(entries)
+			if err != nil {
+				return err
+			}
+		}
+		listType.Filter(&entries)
+		if !includeAll {
+			g, gCtx := errgroup.WithContext(ctx)
+			g.SetLimit(ci.Checkers)
+			for i := range entries {
+				entry := &entries[i]
+				g.Go(func() error {
+					return filterEntry(gCtx, entry)
+				})
+			}
+			err = g.Wait()
+			if err != nil {
+				return err
+			}
+			filteredEntries := entries[:0]
+			for _, entry := range entries {
+				if entry != nil {
+					filteredEntries = append(filteredEntries, entry)
 				}
 			}
 			entries = filteredEntries