From 28da99d5120364f0a0f0b8e802b02c36da89f89c Mon Sep 17 00:00:00 2001
From: David Dworken <david@daviddworken.com>
Date: Sun, 18 Aug 2024 21:48:53 -0700
Subject: [PATCH] Add a basic deep cleaning function for the self-hosted server
 to warn users about cases like #234

---
 backend/server/internal/database/db.go | 51 ++++++++++++++++++++++++++
 backend/server/server.go               | 13 +++++--
 2 files changed, 61 insertions(+), 3 deletions(-)

diff --git a/backend/server/internal/database/db.go b/backend/server/internal/database/db.go
index c02b02f..d33c623 100644
--- a/backend/server/internal/database/db.go
+++ b/backend/server/internal/database/db.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
@@ -427,6 +428,56 @@ func (db *DB) GenerateAndStoreActiveUserStats(ctx context.Context) error {
 	}).Error
 }
 
+func (db *DB) SelfHostedDeepClean(ctx context.Context) error {
+	return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+		runDeletes := os.Getenv("HISHTORY_SELF_HOSTED_DEEP_CLEAN") != ""
+		r := tx.Exec(`
+		CREATE TEMP TABLE temp_inactive_devices AS (
+			SELECT device_id
+			FROM usage_data
+			WHERE last_used <= (now() - INTERVAL '90 days')
+		)
+		`)
+		if r.Error != nil {
+			return fmt.Errorf("failed to create list of inactive users: %w", r.Error)
+		}
+		if runDeletes {
+			r = tx.Raw(`
+			DELETE FROM enc_history_entries WHERE
+				device_id IN (SELECT * FROM temp_inactive_devices)
+			`)
+			if r.Error != nil {
+				return fmt.Errorf("failed to delete entries for inactive devices: %w", r.Error)
+			}
+			r = tx.Raw(`
+			DELETE FROM devices WHERE
+				device_id IN (SELECT * FROM temp_inactive_devices)
+			`)
+			if r.Error != nil {
+				return fmt.Errorf("failed to delete inactive devices: %w", r.Error)
+			}
+		} else {
+			r = tx.Raw(`
+			SELECT COUNT(*)
+			FROM enc_history_entries
+			WHERE device_id IN (SELECT * FROM temp_inactive_devices)
+			`)
+			if r.Error != nil {
+				return fmt.Errorf("failed to count entries for inactive devices: %w", r.Error)
+			}
+			count, err := extractInt64FromRow(r.Row())
+			if err != nil {
+				return fmt.Errorf("failed to extract count of entries for inactive devices: %w", err)
+			}
+			if count > 10_000 {
+				fmt.Printf("WARNING: This server is persisting %d entries for devices that have been offline for more than 90 days. If this is unexpected, set the server environment variable HISHTORY_SELF_HOSTED_DEEP_CLEAN=1 to permanently delete these devices.\n", count)
+			}
+		}
+		fmt.Println("Successfully checked for inactive devices")
+		return nil
+	})
+}
+
 func (db *DB) DeepClean(ctx context.Context) error {
 	err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
 		// Delete entries for users that have one device and are inactive
diff --git a/backend/server/server.go b/backend/server/server.go
index 4df83ec..a2cee93 100644
--- a/backend/server/server.go
+++ b/backend/server/server.go
@@ -128,10 +128,17 @@ func cron(ctx context.Context, db *database.DB, stats *statsd.Client) error {
 	}
 
 	// Run a deep clean less often to cover some more edge cases that hurt DB performance
-	if isProductionEnvironment() && time.Since(LAST_DEEP_CLEAN) > 24*3*time.Hour {
+	if time.Since(LAST_DEEP_CLEAN) > 24*3*time.Hour {
 		LAST_DEEP_CLEAN = time.Now()
-		if err := db.DeepClean(ctx); err != nil {
-			return fmt.Errorf("db.DeepClean: %w", err)
+		if isProductionEnvironment() {
+			if err := db.DeepClean(ctx); err != nil {
+				return fmt.Errorf("db.DeepClean: %w", err)
+			}
+		}
+		if !isProductionEnvironment() && !isTestEnvironment() {
+			if err := db.SelfHostedDeepClean(ctx); err != nil {
+				return fmt.Errorf("db.SelfHostedDeepClean: %w", err)
+			}
 		}
 	}