From bff229713a348a5aabf867b183d73d0f67b401df Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 24 Nov 2024 10:52:28 +0000 Subject: [PATCH] accounting: Add listed stat for number of directory entries listed --- fs/accounting/prometheus.go | 7 +++++++ fs/accounting/stats.go | 19 +++++++++++++++---- fs/accounting/stats_groups.go | 2 ++ fs/list/list.go | 2 ++ fs/operations/lsjson.go | 5 ++++- fs/walk/walk.go | 3 +++ 6 files changed, 33 insertions(+), 5 deletions(-) diff --git a/fs/accounting/prometheus.go b/fs/accounting/prometheus.go index cad792804..5ae07b749 100644 --- a/fs/accounting/prometheus.go +++ b/fs/accounting/prometheus.go @@ -19,6 +19,7 @@ type RcloneCollector struct { deletes *prometheus.Desc deletedDirs *prometheus.Desc renames *prometheus.Desc + listed *prometheus.Desc fatalError *prometheus.Desc retryError *prometheus.Desc } @@ -59,6 +60,10 @@ func NewRcloneCollector(ctx context.Context) *RcloneCollector { "Total number of files renamed", nil, nil, ), + listed: prometheus.NewDesc(namespace+"entries_listed_total", + "Total number of entries listed", + nil, nil, + ), fatalError: prometheus.NewDesc(namespace+"fatal_error", "Whether a fatal error has occurred", nil, nil, @@ -80,6 +85,7 @@ func (c *RcloneCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.deletes ch <- c.deletedDirs ch <- c.renames + ch <- c.listed ch <- c.fatalError ch <- c.retryError } @@ -97,6 +103,7 @@ func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(c.deletes, prometheus.CounterValue, float64(s.deletes)) ch <- prometheus.MustNewConstMetric(c.deletedDirs, prometheus.CounterValue, float64(s.deletedDirs)) ch <- prometheus.MustNewConstMetric(c.renames, prometheus.CounterValue, float64(s.renames)) + ch <- prometheus.MustNewConstMetric(c.listed, prometheus.CounterValue, float64(s.listed)) ch <- prometheus.MustNewConstMetric(c.fatalError, prometheus.GaugeValue, bool2Float(s.fatalError)) ch <- prometheus.MustNewConstMetric(c.retryError, prometheus.GaugeValue, bool2Float(s.retryError)) diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go index c50ef77b0..18249e300 100644 --- a/fs/accounting/stats.go +++ b/fs/accounting/stats.go @@ -46,6 +46,7 @@ type StatsInfo struct { transferring *transferMap transferQueue int transferQueueSize int64 + listed int64 renames int64 renameQueue int renameQueueSize int64 @@ -117,6 +118,7 @@ func (s *StatsInfo) RemoteStats(short bool) (out rc.Params, err error) { out["deletes"] = s.deletes out["deletedDirs"] = s.deletedDirs out["renames"] = s.renames + out["listed"] = s.listed out["elapsedTime"] = time.Since(s.startTime).Seconds() out["serverSideCopies"] = s.serverSideCopies out["serverSideCopyBytes"] = s.serverSideCopyBytes @@ -500,9 +502,9 @@ func (s *StatsInfo) String() string { _, _ = fmt.Fprintf(buf, "Errors: %10d%s\n", s.errors, errorDetails) } - if s.checks != 0 || ts.totalChecks != 0 { - _, _ = fmt.Fprintf(buf, "Checks: %10d / %d, %s\n", - s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks)) + if s.checks != 0 || ts.totalChecks != 0 || s.listed != 0 { + _, _ = fmt.Fprintf(buf, "Checks: %10d / %d, %s, Listed %d\n", + s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks), s.listed) } if s.deletes != 0 || s.deletedDirs != 0 { _, _ = fmt.Fprintf(buf, "Deleted: %10d (files), %d (dirs), %s (freed)\n", s.deletes, s.deletedDirs, fs.SizeSuffix(s.deletesSize).ByteUnit()) @@ -718,7 +720,15 @@ func (s *StatsInfo) Renames(renames int64) int64 { return s.renames } -// ResetCounters sets the counters (bytes, checks, errors, transfers, deletes, renames) to 0 and resets lastError, fatalError and retryError +// Listed updates the stats for listed objects +func (s *StatsInfo) Listed(listed int64) int64 { + s.mu.Lock() + defer s.mu.Unlock() + s.listed += listed + return s.listed +} + +// ResetCounters sets the counters (bytes, checks, errors, transfers, deletes, renames, listed) to 0 and resets lastError, fatalError and retryError func (s *StatsInfo) ResetCounters() { s.mu.Lock() defer s.mu.Unlock() @@ -734,6 +744,7 @@ func (s *StatsInfo) ResetCounters() { s.deletesSize = 0 s.deletedDirs = 0 s.renames = 0 + s.listed = 0 s.startedTransfers = nil s.oldDuration = 0 diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.go index 13236aa52..3c971d831 100644 --- a/fs/accounting/stats_groups.go +++ b/fs/accounting/stats_groups.go @@ -96,6 +96,7 @@ Returns the following values: "fatalError": boolean whether there has been at least one fatal error, "lastError": last error string, "renames" : number of files renamed, + "listed" : number of directory entries listed, "retryError": boolean showing whether there has been at least one non-NoRetryError, "serverSideCopies": number of server side copies done, "serverSideCopyBytes": number bytes server side copied, @@ -383,6 +384,7 @@ func (sg *statsGroups) sum(ctx context.Context) *StatsInfo { sum.transfers += stats.transfers sum.transferring.merge(stats.transferring) sum.transferQueueSize += stats.transferQueueSize + sum.listed += stats.listed sum.renames += stats.renames sum.renameQueue += stats.renameQueue sum.renameQueueSize += stats.renameQueueSize diff --git a/fs/list/list.go b/fs/list/list.go index 50613779d..edd56f889 100644 --- a/fs/list/list.go +++ b/fs/list/list.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/lib/bucket" ) @@ -23,6 +24,7 @@ import ( func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) { // Get unfiltered entries from the fs entries, err = f.List(ctx, dir) + accounting.Stats(ctx).Listed(int64(len(entries))) if err != nil { return nil, err } diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.go index 94d405e2f..8e714efbf 100644 --- a/fs/operations/lsjson.go +++ b/fs/operations/lsjson.go @@ -10,6 +10,7 @@ import ( "github.com/rclone/rclone/backend/crypt" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/walk" ) @@ -283,7 +284,8 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt) return nil, nil } // Check the root directory exists - _, err := fsrc.List(ctx, "") + entries, err := fsrc.List(ctx, "") + accounting.Stats(ctx).Listed(int64(len(entries))) if err != nil { return nil, err } @@ -322,6 +324,7 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt) parent = "" } entries, err := fsrc.List(ctx, parent) + accounting.Stats(ctx).Listed(int64(len(entries))) if err == fs.ErrorDirNotFound { return nil, nil } else if err != nil { diff --git a/fs/walk/walk.go b/fs/walk/walk.go index ab157e3a1..5d64f97d3 100644 --- a/fs/walk/walk.go +++ b/fs/walk/walk.go @@ -12,6 +12,7 @@ import ( "time" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/dirtree" "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/list" @@ -296,6 +297,7 @@ func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType } var mu sync.Mutex err := doListR(ctx, path, func(entries fs.DirEntries) (err error) { + accounting.Stats(ctx).Listed(int64(len(entries))) if synthesizeDirs { err = dm.addEntries(entries) if err != nil { @@ -465,6 +467,7 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo includeDirectory := fi.IncludeDirectory(ctx, f) var mu sync.Mutex err := listR(ctx, startPath, func(entries fs.DirEntries) error { + accounting.Stats(ctx).Listed(int64(len(entries))) mu.Lock() defer mu.Unlock() for _, entry := range entries {