accounting: add limits and listing to stats groups

fs/accounting/stats_groups.go

@@ -13,6 +13,14 @@ const globalStats = "global_stats"
 
 var groups *statsGroups
 
+func listStats(ctx context.Context, in rc.Params) (rc.Params, error) {
+	out := make(rc.Params)
+
+	out["groups"] = groups.names()
+
+	return out, nil
+}
+
 func remoteStats(ctx context.Context, in rc.Params) (rc.Params, error) {
 	// Check to see if we should filter by group.
 	group, err := in.GetString("group")
@@ -125,6 +133,26 @@ Returns the following values:
 			}
 		]
 }
+`,
+	})
+
+	rc.Add(rc.Call{
+		Path:  "core/group_list",
+		Fn:    listStats,
+		Title: "Returns list of stats.",
+		Help: `
+This returns list of stats groups currently in memory. 
+
+Returns the following values:
+` + "```" + `
+{
+	"groups":  an array of group names:
+		[
+			"group1",
+			"group2",
+			...
+		]
+}
 `,
 	})
 }
@@ -180,8 +208,9 @@ func NewStatsGroup(group string) *StatsInfo {
 
 // statsGroups holds a synchronized map of stats
 type statsGroups struct {
-	mu sync.Mutex
-	m  map[string]*StatsInfo
+	mu    sync.Mutex
+	m     map[string]*StatsInfo
+	order []string
 }
 
 // newStatsGroups makes a new statsGroups object
@@ -192,17 +221,24 @@ func newStatsGroups() *statsGroups {
 }
 
 // set marks the stats as belonging to a group
-func (sg *statsGroups) set(group string, acc *StatsInfo) {
+func (sg *statsGroups) set(group string, stats *StatsInfo) {
 	sg.mu.Lock()
 	defer sg.mu.Unlock()
-	sg.m[group] = acc
-}
 
-// clear discards reference to group
-func (sg *statsGroups) clear(group string) {
-	sg.mu.Lock()
-	defer sg.mu.Unlock()
-	delete(sg.m, group)
+	// Limit number of groups kept in memory.
+	if len(sg.order) >= fs.Config.MaxStatsGroups {
+		group := sg.order[0]
+		fs.LogPrintf(fs.LogLevelInfo, nil, "Max number of stats groups reached removing %s", group)
+		delete(sg.m, group)
+		r := (len(sg.order) - fs.Config.MaxStatsGroups) + 1
+		sg.order = sg.order[r:]
+	}
+
+	// Exclude global stats from
+	if group != globalStats {
+		sg.order = append(sg.order, group)
+	}
+	sg.m[group] = stats
 }
 
 // get gets the stats for group, or nil if not found
@@ -216,6 +252,12 @@ func (sg *statsGroups) get(group string) *StatsInfo {
 	return stats
 }
 
+func (sg *statsGroups) names() []string {
+	sg.mu.Lock()
+	defer sg.mu.Unlock()
+	return sg.order
+}
+
 // get gets the stats for group, or nil if not found
 func (sg *statsGroups) sum() *StatsInfo {
 	sg.mu.Lock()

fs/config.go

@@ -88,6 +88,7 @@ type ConfigInfo struct {
 	UseServerModTime       bool
 	MaxTransfer            SizeSuffix
 	MaxBacklog             int
+	MaxStatsGroups         int
 	StatsOneLine           bool
 	StatsOneLineDate       bool   // If we want a date prefix at all
 	StatsOneLineDateFormat string // If we want to customize the prefix
@@ -125,6 +126,7 @@ func NewConfig() *ConfigInfo {
 	c.BufferSize = SizeSuffix(16 << 20)
 	c.UserAgent = "rclone/" + Version
 	c.StreamingUploadCutoff = SizeSuffix(100 * 1024)
+	c.MaxStatsGroups = 1000
 	c.StatsFileNameLength = 45
 	c.AskPassword = true
 	c.TPSLimitBurst = 1

fs/config/configflags/configflags.go

@@ -91,6 +91,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.FVarP(flagSet, &fs.Config.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList)
 	flags.FVarP(flagSet, &fs.Config.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer.")
 	flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.")
+	flags.IntVarP(flagSet, &fs.Config.MaxStatsGroups, "max-stats-groups", "", fs.Config.MaxStatsGroups, "Maximum number of stats groups to keep in memory. On max oldest is discarded.")
 	flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.")
 	flags.BoolVarP(flagSet, &fs.Config.StatsOneLineDate, "stats-one-line-date", "", fs.Config.StatsOneLineDate, "Enables --stats-one-line and add current date/time prefix.")
 	flags.StringVarP(flagSet, &fs.Config.StatsOneLineDateFormat, "stats-one-line-date-format", "", fs.Config.StatsOneLineDateFormat, "Enables --stats-one-line-date and uses custom formatted date. Enclose date string in double quotes (\"). See https://golang.org/pkg/time/#Time.Format")