diff --git a/backend/server/internal/database/db.go b/backend/server/internal/database/db.go index 1afc15a..dcac680 100644 --- a/backend/server/internal/database/db.go +++ b/backend/server/internal/database/db.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "strings" + "time" "github.com/ddworken/hishtory/shared" "github.com/jackc/pgx/v4/stdlib" @@ -51,6 +52,7 @@ func (db *DB) AddDatabaseTables() error { &shared.DumpRequest{}, &shared.DeletionRequest{}, &shared.Feedback{}, + &ActiveUserStats{}, } for _, model := range models { @@ -264,6 +266,74 @@ func (db *DB) Clean(ctx context.Context) error { return nil } +func extractInt64FromRow(row *sql.Row) (int64, error) { + var ret int64 + err := row.Scan(&ret) + if err != nil { + return 0, fmt.Errorf("extractInt64FromRow: %w", err) + } + return ret, nil +} + +type ActiveUserStats struct { + Date time.Time + TotalNumDevices int64 + TotalNumUsers int64 + DailyActiveSubmitUsers int64 + DailyActiveQueryUsers int64 + WeeklyActiveSubmitUsers int64 + WeeklyActiveQueryUsers int64 + DailyInstalls int64 + DailyUninstalls int64 +} + +func (db *DB) GenerateAndStoreActiveUserStats(ctx context.Context) error { + totalNumDevices, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT devices.device_id) FROM devices").Row()) + if err != nil { + return err + } + totalNumUsers, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT devices.user_id) FROM devices").Row()) + if err != nil { + return err + } + dauSubmit, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_used > (now()::date-1)::timestamp").Row()) + if err != nil { + return err + } + dauQuery, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_queried > (now()::date-1)::timestamp").Row()) + if err != nil { + return err + } + wauSubmit, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_used > (now()::date-7)::timestamp").Row()) + if err != nil { + return err + } + wauQuery, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_queried > (now()::date-7)::timestamp").Row()) + if err != nil { + return err + } + dailyInstalls, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT count(distinct device_id) FROM devices WHERE registration_date > (now()::date-1)::timestamp").Row()) + if err != nil { + return err + } + dailyUninstalls, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(*) FROM feedbacks WHERE date > (now()::date-1)::timestamp").Row()) + if err != nil { + return err + } + + return db.Create(ActiveUserStats{ + Date: time.Now(), + TotalNumDevices: totalNumDevices, + TotalNumUsers: totalNumUsers, + DailyActiveSubmitUsers: dauSubmit, + DailyActiveQueryUsers: dauQuery, + WeeklyActiveSubmitUsers: wauSubmit, + WeeklyActiveQueryUsers: wauQuery, + DailyInstalls: dailyInstalls, + DailyUninstalls: dailyUninstalls, + }).Error +} + func (db *DB) DeepClean(ctx context.Context) error { return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { r := tx.Exec(` diff --git a/backend/server/server.go b/backend/server/server.go index bae2833..f9e2708 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -102,19 +102,36 @@ func OpenDB() (*database.DB, error) { return db, nil } +var CRON_COUNTER = 0 + func cron(ctx context.Context, db *database.DB, stats *statsd.Client) error { + // Determine the latest released version of hishtory to serve via the /api/v1/download + // endpoint for hishtory updates. if err := release.UpdateReleaseVersion(); err != nil { return fmt.Errorf("updateReleaseVersion: %w", err) } + // Clean the DB to remove entries that have already been read if err := db.Clean(ctx); err != nil { return fmt.Errorf("db.Clean: %w", err) } + + // Flush out datadog statsd if stats != nil { if err := stats.Flush(); err != nil { return fmt.Errorf("stats.Flush: %w", err) } } + + // Collect and store metrics on active users so we can track trends over time. This doesn't + // have to be run as often, so only run it for a fraction of cron jobs + if CRON_COUNTER%40 == 0 { + if err := db.GenerateAndStoreActiveUserStats(ctx); err != nil { + return fmt.Errorf("db.GenerateAndStoreActiveUserStats: %w", err) + } + } + + CRON_COUNTER += 1 return nil } @@ -123,10 +140,7 @@ func runBackgroundJobs(ctx context.Context, srv *server.Server, db *database.DB, for { err := cron(ctx, db, stats) if err != nil { - fmt.Printf("Cron failure: %v", err) - - // cron no longer panics, panicking here. - panic(err) + panic(fmt.Sprintf("Cron failure: %v", err)) } srv.UpdateReleaseVersion(release.Version, release.BuildUpdateInfo(release.Version)) time.Sleep(10 * time.Minute)