accounting: Add listed stat for number of directory entries listed

fs/accounting/prometheus.go

@@ -19,6 +19,7 @@ type RcloneCollector struct {
 	deletes          *prometheus.Desc
 	deletedDirs      *prometheus.Desc
 	renames          *prometheus.Desc
+	listed           *prometheus.Desc
 	fatalError       *prometheus.Desc
 	retryError       *prometheus.Desc
 }
@@ -59,6 +60,10 @@ func NewRcloneCollector(ctx context.Context) *RcloneCollector {
 			"Total number of files renamed",
 			nil, nil,
 		),
+		listed: prometheus.NewDesc(namespace+"entries_listed_total",
+			"Total number of entries listed",
+			nil, nil,
+		),
 		fatalError: prometheus.NewDesc(namespace+"fatal_error",
 			"Whether a fatal error has occurred",
 			nil, nil,
@@ -80,6 +85,7 @@ func (c *RcloneCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- c.deletes
 	ch <- c.deletedDirs
 	ch <- c.renames
+	ch <- c.listed
 	ch <- c.fatalError
 	ch <- c.retryError
 }
@@ -97,6 +103,7 @@ func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(c.deletes, prometheus.CounterValue, float64(s.deletes))
 	ch <- prometheus.MustNewConstMetric(c.deletedDirs, prometheus.CounterValue, float64(s.deletedDirs))
 	ch <- prometheus.MustNewConstMetric(c.renames, prometheus.CounterValue, float64(s.renames))
+	ch <- prometheus.MustNewConstMetric(c.listed, prometheus.CounterValue, float64(s.listed))
 	ch <- prometheus.MustNewConstMetric(c.fatalError, prometheus.GaugeValue, bool2Float(s.fatalError))
 	ch <- prometheus.MustNewConstMetric(c.retryError, prometheus.GaugeValue, bool2Float(s.retryError))
 

fs/accounting/stats.go

@@ -46,6 +46,7 @@ type StatsInfo struct {
 	transferring        *transferMap
 	transferQueue       int
 	transferQueueSize   int64
+	listed              int64
 	renames             int64
 	renameQueue         int
 	renameQueueSize     int64
@@ -117,6 +118,7 @@ func (s *StatsInfo) RemoteStats(short bool) (out rc.Params, err error) {
 	out["deletes"] = s.deletes
 	out["deletedDirs"] = s.deletedDirs
 	out["renames"] = s.renames
+	out["listed"] = s.listed
 	out["elapsedTime"] = time.Since(s.startTime).Seconds()
 	out["serverSideCopies"] = s.serverSideCopies
 	out["serverSideCopyBytes"] = s.serverSideCopyBytes
@@ -500,9 +502,9 @@ func (s *StatsInfo) String() string {
 			_, _ = fmt.Fprintf(buf, "Errors:        %10d%s\n",
 				s.errors, errorDetails)
 		}
-		if s.checks != 0 || ts.totalChecks != 0 {
+		if s.checks != 0 || ts.totalChecks != 0 || s.listed != 0 {
-			_, _ = fmt.Fprintf(buf, "Checks:        %10d / %d, %s\n",
+			_, _ = fmt.Fprintf(buf, "Checks:        %10d / %d, %s, Listed %d\n",
-				s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks))
+				s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks), s.listed)
 		}
 		if s.deletes != 0 || s.deletedDirs != 0 {
 			_, _ = fmt.Fprintf(buf, "Deleted:       %10d (files), %d (dirs), %s (freed)\n", s.deletes, s.deletedDirs, fs.SizeSuffix(s.deletesSize).ByteUnit())
@@ -718,7 +720,15 @@ func (s *StatsInfo) Renames(renames int64) int64 {
 	return s.renames
 }
 
-// ResetCounters sets the counters (bytes, checks, errors, transfers, deletes, renames) to 0 and resets lastError, fatalError and retryError
+// Listed updates the stats for listed objects
+func (s *StatsInfo) Listed(listed int64) int64 {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.listed += listed
+	return s.listed
+}
+
+// ResetCounters sets the counters (bytes, checks, errors, transfers, deletes, renames, listed) to 0 and resets lastError, fatalError and retryError
 func (s *StatsInfo) ResetCounters() {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -734,6 +744,7 @@ func (s *StatsInfo) ResetCounters() {
 	s.deletesSize = 0
 	s.deletedDirs = 0
 	s.renames = 0
+	s.listed = 0
 	s.startedTransfers = nil
 	s.oldDuration = 0
 

fs/accounting/stats_groups.go

@@ -96,6 +96,7 @@ Returns the following values:
 	"fatalError": boolean whether there has been at least one fatal error,
 	"lastError": last error string,
 	"renames" : number of files renamed,
+	"listed" : number of directory entries listed,
 	"retryError": boolean showing whether there has been at least one non-NoRetryError,
         "serverSideCopies": number of server side copies done,
         "serverSideCopyBytes": number bytes server side copied,
@@ -383,6 +384,7 @@ func (sg *statsGroups) sum(ctx context.Context) *StatsInfo {
 			sum.transfers += stats.transfers
 			sum.transferring.merge(stats.transferring)
 			sum.transferQueueSize += stats.transferQueueSize
+			sum.listed += stats.listed
 			sum.renames += stats.renames
 			sum.renameQueue += stats.renameQueue
 			sum.renameQueueSize += stats.renameQueueSize

fs/list/list.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/lib/bucket"
 )
@@ -23,6 +24,7 @@ import (
 func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
 	// Get unfiltered entries from the fs
 	entries, err = f.List(ctx, dir)
+	accounting.Stats(ctx).Listed(int64(len(entries)))
 	if err != nil {
 		return nil, err
 	}

fs/operations/lsjson.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/rclone/rclone/backend/crypt"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
 )
@@ -283,7 +284,8 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt)
 			return nil, nil
 		}
 		// Check the root directory exists
-		_, err := fsrc.List(ctx, "")
+		entries, err := fsrc.List(ctx, "")
+		accounting.Stats(ctx).Listed(int64(len(entries)))
 		if err != nil {
 			return nil, err
 		}
@@ -322,6 +324,7 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt)
 		parent = ""
 	}
 	entries, err := fsrc.List(ctx, parent)
+	accounting.Stats(ctx).Listed(int64(len(entries)))
 	if err == fs.ErrorDirNotFound {
 		return nil, nil
 	} else if err != nil {

fs/walk/walk.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/dirtree"
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/list"
@@ -296,6 +297,7 @@ func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType
 	}
 	var mu sync.Mutex
 	err := doListR(ctx, path, func(entries fs.DirEntries) (err error) {
+		accounting.Stats(ctx).Listed(int64(len(entries)))
 		if synthesizeDirs {
 			err = dm.addEntries(entries)
 			if err != nil {
@@ -465,6 +467,7 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo
 	includeDirectory := fi.IncludeDirectory(ctx, f)
 	var mu sync.Mutex
 	err := listR(ctx, startPath, func(entries fs.DirEntries) error {
+		accounting.Stats(ctx).Listed(int64(len(entries)))
 		mu.Lock()
 		defer mu.Unlock()
 		for _, entry := range entries {