diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.go index 69ba56b36..90a21db5f 100644 --- a/fs/accounting/stats_groups.go +++ b/fs/accounting/stats_groups.go @@ -13,6 +13,14 @@ const globalStats = "global_stats" var groups *statsGroups +func listStats(ctx context.Context, in rc.Params) (rc.Params, error) { + out := make(rc.Params) + + out["groups"] = groups.names() + + return out, nil +} + func remoteStats(ctx context.Context, in rc.Params) (rc.Params, error) { // Check to see if we should filter by group. group, err := in.GetString("group") @@ -125,6 +133,26 @@ Returns the following values: } ] } +`, + }) + + rc.Add(rc.Call{ + Path: "core/group_list", + Fn: listStats, + Title: "Returns list of stats.", + Help: ` +This returns list of stats groups currently in memory. + +Returns the following values: +` + "```" + ` +{ + "groups": an array of group names: + [ + "group1", + "group2", + ... + ] +} `, }) } @@ -180,8 +208,9 @@ func NewStatsGroup(group string) *StatsInfo { // statsGroups holds a synchronized map of stats type statsGroups struct { - mu sync.Mutex - m map[string]*StatsInfo + mu sync.Mutex + m map[string]*StatsInfo + order []string } // newStatsGroups makes a new statsGroups object @@ -192,17 +221,24 @@ func newStatsGroups() *statsGroups { } // set marks the stats as belonging to a group -func (sg *statsGroups) set(group string, acc *StatsInfo) { +func (sg *statsGroups) set(group string, stats *StatsInfo) { sg.mu.Lock() defer sg.mu.Unlock() - sg.m[group] = acc -} -// clear discards reference to group -func (sg *statsGroups) clear(group string) { - sg.mu.Lock() - defer sg.mu.Unlock() - delete(sg.m, group) + // Limit number of groups kept in memory. + if len(sg.order) >= fs.Config.MaxStatsGroups { + group := sg.order[0] + fs.LogPrintf(fs.LogLevelInfo, nil, "Max number of stats groups reached removing %s", group) + delete(sg.m, group) + r := (len(sg.order) - fs.Config.MaxStatsGroups) + 1 + sg.order = sg.order[r:] + } + + // Exclude global stats from + if group != globalStats { + sg.order = append(sg.order, group) + } + sg.m[group] = stats } // get gets the stats for group, or nil if not found @@ -216,6 +252,12 @@ func (sg *statsGroups) get(group string) *StatsInfo { return stats } +func (sg *statsGroups) names() []string { + sg.mu.Lock() + defer sg.mu.Unlock() + return sg.order +} + // get gets the stats for group, or nil if not found func (sg *statsGroups) sum() *StatsInfo { sg.mu.Lock() diff --git a/fs/config.go b/fs/config.go index e5f5d3605..c94da532d 100644 --- a/fs/config.go +++ b/fs/config.go @@ -88,6 +88,7 @@ type ConfigInfo struct { UseServerModTime bool MaxTransfer SizeSuffix MaxBacklog int + MaxStatsGroups int StatsOneLine bool StatsOneLineDate bool // If we want a date prefix at all StatsOneLineDateFormat string // If we want to customize the prefix @@ -125,6 +126,7 @@ func NewConfig() *ConfigInfo { c.BufferSize = SizeSuffix(16 << 20) c.UserAgent = "rclone/" + Version c.StreamingUploadCutoff = SizeSuffix(100 * 1024) + c.MaxStatsGroups = 1000 c.StatsFileNameLength = 45 c.AskPassword = true c.TPSLimitBurst = 1 diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 718a68d2d..90617c79f 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -91,6 +91,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.FVarP(flagSet, &fs.Config.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList) flags.FVarP(flagSet, &fs.Config.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer.") flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.") + flags.IntVarP(flagSet, &fs.Config.MaxStatsGroups, "max-stats-groups", "", fs.Config.MaxStatsGroups, "Maximum number of stats groups to keep in memory. On max oldest is discarded.") flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.") flags.BoolVarP(flagSet, &fs.Config.StatsOneLineDate, "stats-one-line-date", "", fs.Config.StatsOneLineDate, "Enables --stats-one-line and add current date/time prefix.") flags.StringVarP(flagSet, &fs.Config.StatsOneLineDateFormat, "stats-one-line-date-format", "", fs.Config.StatsOneLineDateFormat, "Enables --stats-one-line-date and uses custom formatted date. Enclose date string in double quotes (\"). See https://golang.org/pkg/time/#Time.Format")