extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-01-03 04:49:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

accounting: Add listed stat for number of directory entries listed

Browse Source
This commit is contained in:
Nick Craig-Wood 2024-11-24 10:52:28 +00:00
parent 65123037d6
commit d48cf0ade1
6 changed files with 33 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
fs/accounting/prometheus.go
View File

@ -19,6 +19,7 @@ type RcloneCollector struct {
deletes *prometheus.Desc
deletedDirs *prometheus.Desc
renames *prometheus.Desc
listed *prometheus.Desc
fatalError *prometheus.Desc
retryError *prometheus.Desc
}
@ -59,6 +60,10 @@ func NewRcloneCollector(ctx context.Context) *RcloneCollector {
"Total number of files renamed",
nil, nil,
),
listed: prometheus.NewDesc(namespace+"entries_listed_total",
"Total number of entries listed",
nil, nil,
),
fatalError: prometheus.NewDesc(namespace+"fatal_error",
"Whether a fatal error has occurred",
nil, nil,
@ -80,6 +85,7 @@ func (c *RcloneCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.deletes
ch <- c.deletedDirs
ch <- c.renames
ch <- c.listed
ch <- c.fatalError
ch <- c.retryError
}
@ -97,6 +103,7 @@ func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.deletes, prometheus.CounterValue, float64(s.deletes))
ch <- prometheus.MustNewConstMetric(c.deletedDirs, prometheus.CounterValue, float64(s.deletedDirs))
ch <- prometheus.MustNewConstMetric(c.renames, prometheus.CounterValue, float64(s.renames))
ch <- prometheus.MustNewConstMetric(c.listed, prometheus.CounterValue, float64(s.listed))
ch <- prometheus.MustNewConstMetric(c.fatalError, prometheus.GaugeValue, bool2Float(s.fatalError))
ch <- prometheus.MustNewConstMetric(c.retryError, prometheus.GaugeValue, bool2Float(s.retryError))

19
fs/accounting/stats.go
View File

@ -45,6 +45,7 @@ type StatsInfo struct {
transferring *transferMap
transferQueue int
transferQueueSize int64
listed int64
renames int64
renameQueue int
renameQueueSize int64
@ -113,6 +114,7 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) {
out["deletes"] = s.deletes
out["deletedDirs"] = s.deletedDirs
out["renames"] = s.renames
out["listed"] = s.listed
out["elapsedTime"] = time.Since(s.startTime).Seconds()
out["serverSideCopies"] = s.serverSideCopies
out["serverSideCopyBytes"] = s.serverSideCopyBytes
@ -461,9 +463,9 @@ func (s *StatsInfo) String() string {
_, _ = fmt.Fprintf(buf, "Errors: %10d%s\n",
s.errors, errorDetails)
}
if s.checks != 0 || ts.totalChecks != 0 {
_, _ = fmt.Fprintf(buf, "Checks: %10d / %d, %s\n",
s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks))
if s.checks != 0 || ts.totalChecks != 0 || s.listed != 0 {
_, _ = fmt.Fprintf(buf, "Checks: %10d / %d, %s, Listed %d\n",
s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks), s.listed)
}
if s.deletes != 0 || s.deletedDirs != 0 {
_, _ = fmt.Fprintf(buf, "Deleted: %10d (files), %d (dirs), %s (freed)\n", s.deletes, s.deletedDirs, fs.SizeSuffix(s.deletesSize).ByteUnit())
@ -679,7 +681,15 @@ func (s *StatsInfo) Renames(renames int64) int64 {
return s.renames
}
// ResetCounters sets the counters (bytes, checks, errors, transfers, deletes, renames) to 0 and resets lastError, fatalError and retryError
// Listed updates the stats for listed objects
func (s *StatsInfo) Listed(listed int64) int64 {
s.mu.Lock()
defer s.mu.Unlock()
s.listed += listed
return s.listed
}
// ResetCounters sets the counters (bytes, checks, errors, transfers, deletes, renames, listed) to 0 and resets lastError, fatalError and retryError
func (s *StatsInfo) ResetCounters() {
s.mu.Lock()
defer s.mu.Unlock()
@ -695,6 +705,7 @@ func (s *StatsInfo) ResetCounters() {
s.deletesSize = 0
s.deletedDirs = 0
s.renames = 0
s.listed = 0
s.startedTransfers = nil
s.oldDuration = 0

2
fs/accounting/stats_groups.go
View File

@ -93,6 +93,7 @@ Returns the following values:
"fatalError": boolean whether there has been at least one fatal error,
"lastError": last error string,
"renames" : number of files renamed,
"listed" : number of directory entries listed,
"retryError": boolean showing whether there has been at least one non-NoRetryError,
"serverSideCopies": number of server side copies done,
"serverSideCopyBytes": number bytes server side copied,
@ -380,6 +381,7 @@ func (sg *statsGroups) sum(ctx context.Context) *StatsInfo {
sum.transfers += stats.transfers
sum.transferring.merge(stats.transferring)
sum.transferQueueSize += stats.transferQueueSize
sum.listed += stats.listed
sum.renames += stats.renames
sum.renameQueue += stats.renameQueue
sum.renameQueueSize += stats.renameQueueSize

2
fs/list/list.go
View File

@ -8,6 +8,7 @@ import (
"strings"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/filter"
)
@ -22,6 +23,7 @@ import (
func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
// Get unfiltered entries from the fs
entries, err = f.List(ctx, dir)
accounting.Stats(ctx).Listed(int64(len(entries)))
if err != nil {
return nil, err
}

5
fs/operations/lsjson.go
View File

@ -10,6 +10,7 @@ import (
"github.com/rclone/rclone/backend/crypt"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/walk"
)
@ -283,7 +284,8 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt)
return nil, nil
}
// Check the root directory exists
_, err := fsrc.List(ctx, "")
entries, err := fsrc.List(ctx, "")
accounting.Stats(ctx).Listed(int64(len(entries)))
if err != nil {
return nil, err
}
@ -322,6 +324,7 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt)
parent = ""
}
entries, err := fsrc.List(ctx, parent)
accounting.Stats(ctx).Listed(int64(len(entries)))
if err == fs.ErrorDirNotFound {
return nil, nil
} else if err != nil {

3
fs/walk/walk.go
View File

@ -12,6 +12,7 @@ import (
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/dirtree"
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/list"
@ -296,6 +297,7 @@ func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType
}
var mu sync.Mutex
err := doListR(ctx, path, func(entries fs.DirEntries) (err error) {
accounting.Stats(ctx).Listed(int64(len(entries)))
if synthesizeDirs {
err = dm.addEntries(entries)
if err != nil {
@ -465,6 +467,7 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo
includeDirectory := fi.IncludeDirectory(ctx, f)
var mu sync.Mutex
err := listR(ctx, startPath, func(entries fs.DirEntries) error {
accounting.Stats(ctx).Listed(int64(len(entries)))
mu.Lock()
defer mu.Unlock()
for _, entry := range entries {