Add offline mode for hiSHtory

README.md

@@ -89,6 +89,21 @@ hishtory config-add displayed-columns git_remote
 If you'd like to disable the control-R integration in your shell, you can do so by running `hishtory config-set enable-control-r false`. 
 </details>
 
+<details>
+<summary>Offline Install</summary>
+If you don't need the ability to sync your shell history, you can install hiSHtory in offline mode. 
+
+Download the latest binary from [Github Releases](https://github.com/ddworken/hishtory/releases), and then run `./hishtory-binary install --offline` to install hiSHtory in a fully offline mode. This disables syncing and it is not possible to re-enable syncing after doing this.
+</details>
+
+<details>
+<summary>Self-Hosting</summary>
+By default, hiSHtory relies on a backend for syncing. All data is end-to-end encrypted, so the backend can't view your history. 
+
+But if you'd like to self-host the hishtory backend, you can! The backend is a simple go binary in `backend/server/server.go` that uses postgres to store data. It reads the connection string for the postgres database from `HISHTORY_POSTGRES_DB`. 
+
+More details coming soon!
+</details>
 
 <details>
 <summary>Uninstalling</summary>

client/client_test.go

@@ -124,6 +124,13 @@ func (z zshTester) ShellName() string {
 
 var shellTesters []shellTester = []shellTester{bashTester{}, zshTester{}}
 
+type OnlineStatus int64
+
+const (
+	Online OnlineStatus = iota
+	Offline
+)
+
 func TestParameterized(t *testing.T) {
 	if skipSlowTests() {
 		shellTesters = shellTesters[:1]
@@ -135,7 +142,8 @@ func TestParameterized(t *testing.T) {
 		t.Run("testExcludeHiddenCommand/"+tester.ShellName(), func(t *testing.T) { testExcludeHiddenCommand(t, tester) })
 		t.Run("testUpdate/"+tester.ShellName(), func(t *testing.T) { testUpdate(t, tester) })
 		t.Run("testAdvancedQuery/"+tester.ShellName(), func(t *testing.T) { testAdvancedQuery(t, tester) })
-		t.Run("testIntegration/"+tester.ShellName(), func(t *testing.T) { testIntegration(t, tester) })
+		t.Run("testIntegration/"+tester.ShellName(), func(t *testing.T) { testIntegration(t, tester, Online) })
+		t.Run("testIntegration/offline/"+tester.ShellName(), func(t *testing.T) { testIntegration(t, tester, Offline) })
 		t.Run("testIntegrationWithNewDevice/"+tester.ShellName(), func(t *testing.T) { testIntegrationWithNewDevice(t, tester) })
 		t.Run("testHishtoryBackgroundSaving/"+tester.ShellName(), func(t *testing.T) { testHishtoryBackgroundSaving(t, tester) })
 		t.Run("testDisplayTable/"+tester.ShellName(), func(t *testing.T) { testDisplayTable(t, tester) })
@@ -149,24 +157,26 @@ func TestParameterized(t *testing.T) {
 		t.Run("testReuploadHistoryEntries/"+tester.ShellName(), func(t *testing.T) { testReuploadHistoryEntries(t, tester) })
 		t.Run("testHishtoryOffline/"+tester.ShellName(), func(t *testing.T) { testHishtoryOffline(t, tester) })
 		t.Run("testInitialHistoryImport/"+tester.ShellName(), func(t *testing.T) { testInitialHistoryImport(t, tester) })
-		t.Run("testLocalRedaction/"+tester.ShellName(), func(t *testing.T) { testLocalRedaction(t, tester) })
+		t.Run("testLocalRedaction/"+tester.ShellName(), func(t *testing.T) { testLocalRedaction(t, tester, Online) })
+		t.Run("testLocalRedaction/offline/"+tester.ShellName(), func(t *testing.T) { testLocalRedaction(t, tester, Offline) })
 		t.Run("testRemoteRedaction/"+tester.ShellName(), func(t *testing.T) { testRemoteRedaction(t, tester) })
 		t.Run("testMultipleUsers/"+tester.ShellName(), func(t *testing.T) { testMultipleUsers(t, tester) })
 		t.Run("testConfigGetSet/"+tester.ShellName(), func(t *testing.T) { testConfigGetSet(t, tester) })
-		t.Run("testControlR/"+tester.ShellName(), func(t *testing.T) { testControlR(t, tester, tester.ShellName()) })
+		t.Run("testControlR/"+tester.ShellName(), func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Online) })
+		t.Run("testControlR/offline/"+tester.ShellName(), func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Offline) })
 		t.Run("testHandleUpgradedFeatures/"+tester.ShellName(), func(t *testing.T) { testHandleUpgradedFeatures(t, tester) })
 		t.Run("testCustomColumns/"+tester.ShellName(), func(t *testing.T) { testCustomColumns(t, tester) })
 		t.Run("testUninstall/"+tester.ShellName(), func(t *testing.T) { testUninstall(t, tester) })
 	}
-	t.Run("testControlR/fish", func(t *testing.T) { testControlR(t, bashTester{}, "fish") })
+	t.Run("testControlR/fish", func(t *testing.T) { testControlR(t, bashTester{}, "fish", Online) })
 }
 
-func testIntegration(t *testing.T, tester shellTester) {
+func testIntegration(t *testing.T, tester shellTester, onlineStatus OnlineStatus) {
 	// Set up
 	defer testutils.BackupAndRestore(t)()
 
 	// Run the test
-	testBasicUserFlow(t, tester)
+	testBasicUserFlow(t, tester, onlineStatus)
 }
 
 func testIntegrationWithNewDevice(t *testing.T, tester shellTester) {
@@ -174,7 +184,7 @@ func testIntegrationWithNewDevice(t *testing.T, tester shellTester) {
 	defer testutils.BackupAndRestore(t)()
 
 	// Run the test
-	userSecret := testBasicUserFlow(t, tester)
+	userSecret := testBasicUserFlow(t, tester, Online)
 
 	// Clear all local state
 	testutils.ResetLocalState(t)
@@ -183,7 +193,7 @@ func testIntegrationWithNewDevice(t *testing.T, tester shellTester) {
 	installHishtory(t, tester, userSecret)
 
 	// Querying should show the history from the previous run
-	out := hishtoryQuery(t, tester, "")
+	out := tester.RunInteractiveShell(t, `hishtory query`)
 	expected := []string{"echo thisisrecorded", "hishtory enable", "echo bar", "echo foo", "ls /foo", "ls /bar", "ls /a"}
 	for _, item := range expected {
 		if !strings.Contains(out, item) {
@@ -295,9 +305,28 @@ func installHishtory(t *testing.T, tester shellTester, userSecret string) string
 	return matches[1]
 }
 
-func testBasicUserFlow(t *testing.T, tester shellTester) string {
+func initFromOnlineStatus(t *testing.T, tester shellTester, onlineStatus OnlineStatus) string {
+	if onlineStatus == Online {
+		return installHishtory(t, tester, "")
+	} else {
+		return installHishtory(t, tester, "--offline")
+	}
+}
+
+func assertOnlineStatus(t *testing.T, onlineStatus OnlineStatus) {
+	config := hctx.GetConf(hctx.MakeContext())
+	if onlineStatus == Online && config.IsOffline == true {
+		t.Fatalf("We're supposed to be online, yet config.IsOffline=%#v (config=%#v)", config.IsOffline, config)
+	}
+	if onlineStatus == Offline && config.IsOffline == false {
+		t.Fatalf("We're supposed to be offline, yet config.IsOffline=%#v (config=%#v)", config.IsOffline, config)
+	}
+}
+
+func testBasicUserFlow(t *testing.T, tester shellTester, onlineStatus OnlineStatus) string {
 	// Test install
-	userSecret := installHishtory(t, tester, "")
+	userSecret := initFromOnlineStatus(t, tester, onlineStatus)
+	assertOnlineStatus(t, onlineStatus)
 
 	// Test the status subcommand
 	out := tester.RunInteractiveShell(t, `hishtory status`)
@@ -319,13 +348,15 @@ func testBasicUserFlow(t *testing.T, tester shellTester) string {
 	}
 
 	// Test the banner
-	os.Setenv("FORCED_BANNER", "HELLO_FROM_SERVER")
+	if onlineStatus == Online {
-	defer os.Setenv("FORCED_BANNER", "")
+		os.Setenv("FORCED_BANNER", "HELLO_FROM_SERVER")
-	out = hishtoryQuery(t, tester, "")
+		defer os.Setenv("FORCED_BANNER", "")
-	if !strings.Contains(out, "HELLO_FROM_SERVER\nHostname") {
+		out = hishtoryQuery(t, tester, "")
-		t.Fatalf("hishtory query didn't show the banner message! out=%#v", out)
+		if !strings.Contains(out, "HELLO_FROM_SERVER\nHostname") {
+			t.Fatalf("hishtory query didn't show the banner message! out=%#v", out)
+		}
+		os.Setenv("FORCED_BANNER", "")
 	}
-	os.Setenv("FORCED_BANNER", "")
 
 	// Test recording commands
 	out, err = tester.RunInteractiveShellRelaxed(t, `ls /a
@@ -1491,12 +1522,11 @@ echo %v-bar`, randomCmdUuid, randomCmdUuid)
 	}
 }
 
-func testLocalRedaction(t *testing.T, tester shellTester) {
+func testLocalRedaction(t *testing.T, tester shellTester, onlineStatus OnlineStatus) {
 	// Setup
 	defer testutils.BackupAndRestore(t)()
-
+	initFromOnlineStatus(t, tester, onlineStatus)
-	// Install hishtory
+	assertOnlineStatus(t, onlineStatus)
-	installHishtory(t, tester, "")
 
 	// Record some commands
 	randomCmdUuid := uuid.Must(uuid.NewRandom()).String()
@@ -1855,10 +1885,11 @@ func captureTerminalOutputWithShellNameAndDimensions(t *testing.T, tester shellT
 	return strings.TrimSpace(tester.RunInteractiveShell(t, fullCommand))
 }
 
-func testControlR(t *testing.T, tester shellTester, shellName string) {
+func testControlR(t *testing.T, tester shellTester, shellName string, onlineStatus OnlineStatus) {
 	// Setup
 	defer testutils.BackupAndRestore(t)()
-	installHishtory(t, tester, "")
+	initFromOnlineStatus(t, tester, onlineStatus)
+	assertOnlineStatus(t, onlineStatus)
 
 	// Disable recording so that all our testing commands don't get recorded
 	_, _ = tester.RunInteractiveShellRelaxed(t, ` hishtory disable`)

client/hctx/hctx.go

@@ -161,6 +161,8 @@ type ClientConfig struct {
 	DisplayedColumns []string `json:"displayed_columns"`
 	// Custom columns
 	CustomColumns []CustomColumnDefinition `json:"custom_columns"`
+	// Whether this is an offline instance of hishtory with no syncing
+	IsOffline bool `json:"is_offline"`
 }
 
 type CustomColumnDefinition struct {

client/lib/lib.go

@@ -334,8 +334,16 @@ func shouldSkipHiddenCommand(ctx *context.Context, historyLine string) (bool, er
 
 func Setup(args []string) error {
 	userSecret := uuid.Must(uuid.NewRandom()).String()
+	isOffline := false
 	if len(args) > 2 && args[2] != "" {
-		userSecret = args[2]
+		if args[2] == "--offline" {
+			isOffline = true
+		} else {
+			if args[2][0] == '-' {
+				return fmt.Errorf("refusing to set user secret to %#v since it looks like a flag", args[2])
+			}
+			userSecret = args[2]
+		}
 	}
 	fmt.Println("Setting secret hishtory key to " + string(userSecret))
 
@@ -345,6 +353,7 @@ func Setup(args []string) error {
 	config.IsEnabled = true
 	config.DeviceId = uuid.Must(uuid.NewRandom()).String()
 	config.ControlRSearchEnabled = true
+	config.IsOffline = isOffline
 	err := hctx.SetConfig(config)
 	if err != nil {
 		return fmt.Errorf("failed to persist config to disk: %v", err)
@@ -358,6 +367,9 @@ func Setup(args []string) error {
 	db.Exec("DELETE FROM history_entries")
 
 	// Bootstrap from remote date
+	if config.IsOffline {
+		return nil
+	}
 	_, err = ApiGet("/api/v1/register?user_id=" + data.UserId(userSecret) + "&device_id=" + config.DeviceId)
 	if err != nil {
 		return fmt.Errorf("failed to register device with backend: %v", err)
@@ -1217,10 +1229,10 @@ func ReliableDbCreate(db *gorm.DB, entry interface{}) error {
 					return nil
 				}
 			}
-			return err
+			return fmt.Errorf("unrecoverable sqlite error: %v", err)
 		}
 		if err != nil && err.Error() != "database is locked (5) (SQLITE_BUSY)" {
-			return err
+			return fmt.Errorf("unrecoverable sqlite error: %v", err)
 		}
 	}
 	return fmt.Errorf("failed to create DB entry even with %d retries: %v", i, err)
@@ -1287,6 +1299,9 @@ func Redact(ctx *context.Context, query string, force bool) error {
 
 func deleteOnRemoteInstances(ctx *context.Context, historyEntries []*data.HistoryEntry) error {
 	config := hctx.GetConf(ctx)
+	if config.IsOffline {
+		return nil
+	}
 
 	var deletionRequest shared.DeletionRequest
 	deletionRequest.SendTime = time.Now()
@@ -1308,6 +1323,9 @@ func deleteOnRemoteInstances(ctx *context.Context, historyEntries []*data.Histor
 
 func Reupload(ctx *context.Context) error {
 	config := hctx.GetConf(ctx)
+	if config.IsOffline {
+		return nil
+	}
 	entries, err := Search(ctx, hctx.GetDb(ctx), "", 0)
 	if err != nil {
 		return fmt.Errorf("failed to reupload due to failed search: %v", err)
@@ -1340,6 +1358,9 @@ func chunks[k any](slice []k, chunkSize int) [][]k {
 func RetrieveAdditionalEntriesFromRemote(ctx *context.Context) error {
 	db := hctx.GetDb(ctx)
 	config := hctx.GetConf(ctx)
+	if config.IsOffline {
+		return nil
+	}
 	respBody, err := ApiGet("/api/v1/query?device_id=" + config.DeviceId + "&user_id=" + data.UserId(config.UserSecret))
 	if IsOfflineError(err) {
 		return nil
@@ -1364,7 +1385,9 @@ func RetrieveAdditionalEntriesFromRemote(ctx *context.Context) error {
 
 func ProcessDeletionRequests(ctx *context.Context) error {
 	config := hctx.GetConf(ctx)
-
+	if config.IsOffline {
+		return nil
+	}
 	resp, err := ApiGet("/api/v1/get-deletion-requests?user_id=" + data.UserId(config.UserSecret) + "&device_id=" + config.DeviceId)
 	if IsOfflineError(err) {
 		return nil
@@ -1391,6 +1414,9 @@ func ProcessDeletionRequests(ctx *context.Context) error {
 
 func GetBanner(ctx *context.Context, gitCommit string) ([]byte, error) {
 	config := hctx.GetConf(ctx)
+	if config.IsOffline {
+		return []byte{}, nil
+	}
 	url := "/api/v1/banner?commit_hash=" + gitCommit + "&user_id=" + data.UserId(config.UserSecret) + "&device_id=" + config.DeviceId + "&version=" + Version + "&forced_banner=" + os.Getenv("FORCED_BANNER")
 	return ApiGet(url)
 }

client/lib/lib_test.go

@@ -32,6 +32,34 @@ func TestSetup(t *testing.T) {
 	if len(data) < 10 {
 		t.Fatalf("hishtory secret has unexpected length: %d", len(data))
 	}
+	config := hctx.GetConf(hctx.MakeContext())
+	if config.IsOffline != false {
+		t.Fatalf("hishtory config should have been offline")
+	}
+}
+
+func TestSetupOffline(t *testing.T) {
+	defer testutils.BackupAndRestore(t)()
+	defer testutils.RunTestServer()()
+
+	homedir, err := os.UserHomeDir()
+	testutils.Check(t, err)
+	if _, err := os.Stat(path.Join(homedir, data.HISHTORY_PATH, data.CONFIG_PATH)); err == nil {
+		t.Fatalf("hishtory secret file already exists!")
+	}
+	testutils.Check(t, Setup([]string{"", "", "--offline"}))
+	if _, err := os.Stat(path.Join(homedir, data.HISHTORY_PATH, data.CONFIG_PATH)); err != nil {
+		t.Fatalf("hishtory secret file does not exist after Setup()!")
+	}
+	data, err := os.ReadFile(path.Join(homedir, data.HISHTORY_PATH, data.CONFIG_PATH))
+	testutils.Check(t, err)
+	if len(data) < 10 {
+		t.Fatalf("hishtory secret has unexpected length: %d", len(data))
+	}
+	config := hctx.GetConf(hctx.MakeContext())
+	if config.IsOffline != true {
+		t.Fatalf("hishtory config should have been offline, actual=%#v", string(data))
+	}
 }
 
 func TestBuildHistoryEntry(t *testing.T) {

hishtory.go

@@ -287,6 +287,9 @@ func printDumpStatus(config hctx.ClientConfig) {
 }
 
 func getDumpRequests(config hctx.ClientConfig) ([]*shared.DumpRequest, error) {
+	if config.IsOffline {
+		return make([]*shared.DumpRequest, 0), nil
+	}
 	resp, err := lib.ApiGet("/api/v1/get-dump-requests?user_id=" + data.UserId(config.UserSecret) + "&device_id=" + config.DeviceId)
 	if lib.IsOfflineError(err) {
 		return []*shared.DumpRequest{}, nil
@@ -334,6 +337,9 @@ func maybeUploadSkippedHistoryEntries(ctx *context.Context) error {
 	if !config.HaveMissedUploads {
 		return nil
 	}
+	if config.IsOffline {
+		return nil
+	}
 
 	// Upload the missing entries
 	db := hctx.GetDb(ctx)
@@ -382,19 +388,21 @@ func saveHistoryEntry(ctx *context.Context) {
 	lib.CheckFatalError(err)
 
 	// Persist it remotely
-	jsonValue, err := lib.EncryptAndMarshal(config, []*data.HistoryEntry{entry})
+	if !config.IsOffline {
-	lib.CheckFatalError(err)
+		jsonValue, err := lib.EncryptAndMarshal(config, []*data.HistoryEntry{entry})
-	_, err = lib.ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
+		lib.CheckFatalError(err)
-	if err != nil {
+		_, err = lib.ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
-		if lib.IsOfflineError(err) {
+		if err != nil {
-			hctx.GetLogger().Printf("Failed to remotely persist hishtory entry because the device is offline!")
+			if lib.IsOfflineError(err) {
-			if !config.HaveMissedUploads {
+				hctx.GetLogger().Printf("Failed to remotely persist hishtory entry because the device is offline!")
-				config.HaveMissedUploads = true
+				if !config.HaveMissedUploads {
-				config.MissedUploadTimestamp = time.Now().Unix()
+					config.HaveMissedUploads = true
-				lib.CheckFatalError(hctx.SetConfig(config))
+					config.MissedUploadTimestamp = time.Now().Unix()
+					lib.CheckFatalError(hctx.SetConfig(config))
+				}
+			} else {
+				lib.CheckFatalError(err)
 			}
-		} else {
-			lib.CheckFatalError(err)
 		}
 	}
 
@@ -422,8 +430,10 @@ func saveHistoryEntry(ctx *context.Context) {
 		reqBody, err := json.Marshal(encEntries)
 		lib.CheckFatalError(err)
 		for _, dumpRequest := range dumpRequests {
-			_, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId+"&source_device_id="+config.DeviceId, "application/json", reqBody)
+			if !config.IsOffline {
-			lib.CheckFatalError(err)
+				_, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId+"&source_device_id="+config.DeviceId, "application/json", reqBody)
+				lib.CheckFatalError(err)
+			}
 		}
 	}
 }

shared/testutils/testutils.go

@@ -29,13 +29,7 @@ func ResetLocalState(t *testing.T) {
 		t.Fatalf("failed to retrieve homedir: %v", err)
 	}
 
-	_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, data.DB_PATH))
+	_ = os.RemoveAll(path.Join(homedir, data.HISHTORY_PATH))
-	_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, DB_WAL_PATH))
-	_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, data.CONFIG_PATH))
-	_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, "hishtory"))
-	_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, "config.sh"))
-	_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, "config.zsh"))
-	_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, "config.fish"))
 }
 
 func BackupAndRestore(t *testing.T) func() {
@@ -43,7 +37,10 @@ func BackupAndRestore(t *testing.T) func() {
 }
 
 func getBackPath(file, id string) string {
-	return strings.Replace(file, data.HISHTORY_PATH, data.HISHTORY_PATH+".test", 1) + id
+	if strings.Contains(file, "/"+data.HISHTORY_PATH+"/") {
+		return strings.Replace(file, data.HISHTORY_PATH, data.HISHTORY_PATH+".test", 1) + id
+	}
+	return file + ".bak" + id
 }
 
 func BackupAndRestoreWithId(t *testing.T, id string) func() {