Add backend table to track daily/weekly active user stats

backend/server/internal/database/db.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/ddworken/hishtory/shared"
 	"github.com/jackc/pgx/v4/stdlib"
@@ -51,6 +52,7 @@ func (db *DB) AddDatabaseTables() error {
 		&shared.DumpRequest{},
 		&shared.DeletionRequest{},
 		&shared.Feedback{},
+		&ActiveUserStats{},
 	}
 
 	for _, model := range models {
@@ -264,6 +266,74 @@ func (db *DB) Clean(ctx context.Context) error {
 	return nil
 }
 
+func extractInt64FromRow(row *sql.Row) (int64, error) {
+	var ret int64
+	err := row.Scan(&ret)
+	if err != nil {
+		return 0, fmt.Errorf("extractInt64FromRow: %w", err)
+	}
+	return ret, nil
+}
+
+type ActiveUserStats struct {
+	Date                    time.Time
+	TotalNumDevices         int64
+	TotalNumUsers           int64
+	DailyActiveSubmitUsers  int64
+	DailyActiveQueryUsers   int64
+	WeeklyActiveSubmitUsers int64
+	WeeklyActiveQueryUsers  int64
+	DailyInstalls           int64
+	DailyUninstalls         int64
+}
+
+func (db *DB) GenerateAndStoreActiveUserStats(ctx context.Context) error {
+	totalNumDevices, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT devices.device_id) FROM devices").Row())
+	if err != nil {
+		return err
+	}
+	totalNumUsers, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT devices.user_id) FROM devices").Row())
+	if err != nil {
+		return err
+	}
+	dauSubmit, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_used > (now()::date-1)::timestamp").Row())
+	if err != nil {
+		return err
+	}
+	dauQuery, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_queried > (now()::date-1)::timestamp").Row())
+	if err != nil {
+		return err
+	}
+	wauSubmit, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_used > (now()::date-7)::timestamp").Row())
+	if err != nil {
+		return err
+	}
+	wauQuery, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_queried > (now()::date-7)::timestamp").Row())
+	if err != nil {
+		return err
+	}
+	dailyInstalls, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT count(distinct device_id) FROM devices WHERE registration_date > (now()::date-1)::timestamp").Row())
+	if err != nil {
+		return err
+	}
+	dailyUninstalls, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(*) FROM feedbacks WHERE date > (now()::date-1)::timestamp").Row())
+	if err != nil {
+		return err
+	}
+
+	return db.Create(ActiveUserStats{
+		Date:                    time.Now(),
+		TotalNumDevices:         totalNumDevices,
+		TotalNumUsers:           totalNumUsers,
+		DailyActiveSubmitUsers:  dauSubmit,
+		DailyActiveQueryUsers:   dauQuery,
+		WeeklyActiveSubmitUsers: wauSubmit,
+		WeeklyActiveQueryUsers:  wauQuery,
+		DailyInstalls:           dailyInstalls,
+		DailyUninstalls:         dailyUninstalls,
+	}).Error
+}
+
 func (db *DB) DeepClean(ctx context.Context) error {
 	return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
 		r := tx.Exec(`

backend/server/server.go

@@ -102,19 +102,36 @@ func OpenDB() (*database.DB, error) {
 	return db, nil
 }
 
+var CRON_COUNTER = 0
+
 func cron(ctx context.Context, db *database.DB, stats *statsd.Client) error {
+	// Determine the latest released version of hishtory to serve via the /api/v1/download
+	// endpoint for hishtory updates.
 	if err := release.UpdateReleaseVersion(); err != nil {
 		return fmt.Errorf("updateReleaseVersion: %w", err)
 	}
 
+	// Clean the DB to remove entries that have already been read
 	if err := db.Clean(ctx); err != nil {
 		return fmt.Errorf("db.Clean: %w", err)
 	}
+
+	// Flush out datadog statsd
 	if stats != nil {
 		if err := stats.Flush(); err != nil {
 			return fmt.Errorf("stats.Flush: %w", err)
 		}
 	}
+
+	// Collect and store metrics on active users so we can track trends over time. This doesn't
+	// have to be run as often, so only run it for a fraction of cron jobs
+	if CRON_COUNTER%40 == 0 {
+		if err := db.GenerateAndStoreActiveUserStats(ctx); err != nil {
+			return fmt.Errorf("db.GenerateAndStoreActiveUserStats: %w", err)
+		}
+	}
+
+	CRON_COUNTER += 1
 	return nil
 }
 
@@ -123,10 +140,7 @@ func runBackgroundJobs(ctx context.Context, srv *server.Server, db *database.DB,
 	for {
 		err := cron(ctx, db, stats)
 		if err != nil {
-			fmt.Printf("Cron failure: %v", err)
-
-			// cron no longer panics, panicking here.
-			panic(err)
+			panic(fmt.Sprintf("Cron failure: %v", err))
 		}
 		srv.UpdateReleaseVersion(release.Version, release.BuildUpdateInfo(release.Version))
 		time.Sleep(10 * time.Minute)