extern/hishtory
1
0
Fork 0
mirror of https://github.com/ddworken/hishtory.git synced 2024-11-22 16:24:00 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add offline mode for hiSHtory

Browse Source
This commit is contained in:
David Dworken 2022-11-03 13:16:45 -07:00
parent 0f01dd614c
commit 185d2739c7
No known key found for this signature in database
7 changed files with 157 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
README.md
View File

@ -89,6 +89,21 @@ hishtory config-add displayed-columns git_remote
If you'd like to disable the control-R integration in your shell, you can do so by running `hishtory config-set enable-control-r false`.
</details>
<details>
<summary>Offline Install</summary>
If you don't need the ability to sync your shell history, you can install hiSHtory in offline mode.
Download the latest binary from [Github Releases](https://github.com/ddworken/hishtory/releases), and then run `./hishtory-binary install --offline` to install hiSHtory in a fully offline mode. This disables syncing and it is not possible to re-enable syncing after doing this.
</details>
<details>
<summary>Self-Hosting</summary>
By default, hiSHtory relies on a backend for syncing. All data is end-to-end encrypted, so the backend can't view your history.
But if you'd like to self-host the hishtory backend, you can! The backend is a simple go binary in `backend/server/server.go` that uses postgres to store data. It reads the connection string for the postgres database from `HISHTORY_POSTGRES_DB`.
More details coming soon!
</details>
<details>
<summary>Uninstalling</summary>

75
client/client_test.go
View File

@ -124,6 +124,13 @@ func (z zshTester) ShellName() string {
var shellTesters []shellTester = []shellTester{bashTester{}, zshTester{}}
type OnlineStatus int64
const (
Online OnlineStatus = iota
Offline
)
func TestParameterized(t *testing.T) {
if skipSlowTests() {
shellTesters = shellTesters[:1]
@ -135,7 +142,8 @@ func TestParameterized(t *testing.T) {
t.Run("testExcludeHiddenCommand/"+tester.ShellName(), func(t *testing.T) { testExcludeHiddenCommand(t, tester) })
t.Run("testUpdate/"+tester.ShellName(), func(t *testing.T) { testUpdate(t, tester) })
t.Run("testAdvancedQuery/"+tester.ShellName(), func(t *testing.T) { testAdvancedQuery(t, tester) })
t.Run("testIntegration/"+tester.ShellName(), func(t *testing.T) { testIntegration(t, tester) })
t.Run("testIntegration/"+tester.ShellName(), func(t *testing.T) { testIntegration(t, tester, Online) })
t.Run("testIntegration/offline/"+tester.ShellName(), func(t *testing.T) { testIntegration(t, tester, Offline) })
t.Run("testIntegrationWithNewDevice/"+tester.ShellName(), func(t *testing.T) { testIntegrationWithNewDevice(t, tester) })
t.Run("testHishtoryBackgroundSaving/"+tester.ShellName(), func(t *testing.T) { testHishtoryBackgroundSaving(t, tester) })
t.Run("testDisplayTable/"+tester.ShellName(), func(t *testing.T) { testDisplayTable(t, tester) })
@ -149,24 +157,26 @@ func TestParameterized(t *testing.T) {
t.Run("testReuploadHistoryEntries/"+tester.ShellName(), func(t *testing.T) { testReuploadHistoryEntries(t, tester) })
t.Run("testHishtoryOffline/"+tester.ShellName(), func(t *testing.T) { testHishtoryOffline(t, tester) })
t.Run("testInitialHistoryImport/"+tester.ShellName(), func(t *testing.T) { testInitialHistoryImport(t, tester) })
t.Run("testLocalRedaction/"+tester.ShellName(), func(t *testing.T) { testLocalRedaction(t, tester) })
t.Run("testLocalRedaction/"+tester.ShellName(), func(t *testing.T) { testLocalRedaction(t, tester, Online) })
t.Run("testLocalRedaction/offline/"+tester.ShellName(), func(t *testing.T) { testLocalRedaction(t, tester, Offline) })
t.Run("testRemoteRedaction/"+tester.ShellName(), func(t *testing.T) { testRemoteRedaction(t, tester) })
t.Run("testMultipleUsers/"+tester.ShellName(), func(t *testing.T) { testMultipleUsers(t, tester) })
t.Run("testConfigGetSet/"+tester.ShellName(), func(t *testing.T) { testConfigGetSet(t, tester) })
t.Run("testControlR/"+tester.ShellName(), func(t *testing.T) { testControlR(t, tester, tester.ShellName()) })
t.Run("testControlR/"+tester.ShellName(), func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Online) })
t.Run("testControlR/offline/"+tester.ShellName(), func(t *testing.T) { testControlR(t, tester, tester.ShellName(), Offline) })
t.Run("testHandleUpgradedFeatures/"+tester.ShellName(), func(t *testing.T) { testHandleUpgradedFeatures(t, tester) })
t.Run("testCustomColumns/"+tester.ShellName(), func(t *testing.T) { testCustomColumns(t, tester) })
t.Run("testUninstall/"+tester.ShellName(), func(t *testing.T) { testUninstall(t, tester) })
}
t.Run("testControlR/fish", func(t *testing.T) { testControlR(t, bashTester{}, "fish") })
t.Run("testControlR/fish", func(t *testing.T) { testControlR(t, bashTester{}, "fish", Online) })
}
func testIntegration(t *testing.T, tester shellTester) {
func testIntegration(t *testing.T, tester shellTester, onlineStatus OnlineStatus) {
// Set up
defer testutils.BackupAndRestore(t)()
// Run the test
testBasicUserFlow(t, tester)
testBasicUserFlow(t, tester, onlineStatus)
}
func testIntegrationWithNewDevice(t *testing.T, tester shellTester) {
@ -174,7 +184,7 @@ func testIntegrationWithNewDevice(t *testing.T, tester shellTester) {
defer testutils.BackupAndRestore(t)()
// Run the test
userSecret := testBasicUserFlow(t, tester)
userSecret := testBasicUserFlow(t, tester, Online)
// Clear all local state
testutils.ResetLocalState(t)
@ -183,7 +193,7 @@ func testIntegrationWithNewDevice(t *testing.T, tester shellTester) {
installHishtory(t, tester, userSecret)
// Querying should show the history from the previous run
out := hishtoryQuery(t, tester, "")
out := tester.RunInteractiveShell(t, `hishtory query`)
expected := []string{"echo thisisrecorded", "hishtory enable", "echo bar", "echo foo", "ls /foo", "ls /bar", "ls /a"}
for _, item := range expected {
if !strings.Contains(out, item) {
@ -295,9 +305,28 @@ func installHishtory(t *testing.T, tester shellTester, userSecret string) string
return matches[1]
}
func testBasicUserFlow(t *testing.T, tester shellTester) string {
func initFromOnlineStatus(t *testing.T, tester shellTester, onlineStatus OnlineStatus) string {
if onlineStatus == Online {
return installHishtory(t, tester, "")
} else {
return installHishtory(t, tester, "--offline")
}
}
func assertOnlineStatus(t *testing.T, onlineStatus OnlineStatus) {
config := hctx.GetConf(hctx.MakeContext())
if onlineStatus == Online && config.IsOffline == true {
t.Fatalf("We're supposed to be online, yet config.IsOffline=%#v (config=%#v)", config.IsOffline, config)
}
if onlineStatus == Offline && config.IsOffline == false {
t.Fatalf("We're supposed to be offline, yet config.IsOffline=%#v (config=%#v)", config.IsOffline, config)
}
}
func testBasicUserFlow(t *testing.T, tester shellTester, onlineStatus OnlineStatus) string {
// Test install
userSecret := installHishtory(t, tester, "")
userSecret := initFromOnlineStatus(t, tester, onlineStatus)
assertOnlineStatus(t, onlineStatus)
// Test the status subcommand
out := tester.RunInteractiveShell(t, `hishtory status`)
@ -319,13 +348,15 @@ func testBasicUserFlow(t *testing.T, tester shellTester) string {
}
// Test the banner
os.Setenv("FORCED_BANNER", "HELLO_FROM_SERVER")
defer os.Setenv("FORCED_BANNER", "")
out = hishtoryQuery(t, tester, "")
if !strings.Contains(out, "HELLO_FROM_SERVER\nHostname") {
t.Fatalf("hishtory query didn't show the banner message! out=%#v", out)
if onlineStatus == Online {
os.Setenv("FORCED_BANNER", "HELLO_FROM_SERVER")
defer os.Setenv("FORCED_BANNER", "")
out = hishtoryQuery(t, tester, "")
if !strings.Contains(out, "HELLO_FROM_SERVER\nHostname") {
t.Fatalf("hishtory query didn't show the banner message! out=%#v", out)
}
os.Setenv("FORCED_BANNER", "")
}
os.Setenv("FORCED_BANNER", "")
// Test recording commands
out, err = tester.RunInteractiveShellRelaxed(t, `ls /a
@ -1491,12 +1522,11 @@ echo %v-bar`, randomCmdUuid, randomCmdUuid)
}
}
func testLocalRedaction(t *testing.T, tester shellTester) {
func testLocalRedaction(t *testing.T, tester shellTester, onlineStatus OnlineStatus) {
// Setup
defer testutils.BackupAndRestore(t)()
// Install hishtory
installHishtory(t, tester, "")
initFromOnlineStatus(t, tester, onlineStatus)
assertOnlineStatus(t, onlineStatus)
// Record some commands
randomCmdUuid := uuid.Must(uuid.NewRandom()).String()
@ -1855,10 +1885,11 @@ func captureTerminalOutputWithShellNameAndDimensions(t *testing.T, tester shellT
return strings.TrimSpace(tester.RunInteractiveShell(t, fullCommand))
}
func testControlR(t *testing.T, tester shellTester, shellName string) {
func testControlR(t *testing.T, tester shellTester, shellName string, onlineStatus OnlineStatus) {
// Setup
defer testutils.BackupAndRestore(t)()
installHishtory(t, tester, "")
initFromOnlineStatus(t, tester, onlineStatus)
assertOnlineStatus(t, onlineStatus)
// Disable recording so that all our testing commands don't get recorded
_, _ = tester.RunInteractiveShellRelaxed(t, ` hishtory disable`)

2
client/hctx/hctx.go
View File

@ -161,6 +161,8 @@ type ClientConfig struct {
DisplayedColumns []string `json:"displayed_columns"`
// Custom columns
CustomColumns []CustomColumnDefinition `json:"custom_columns"`
// Whether this is an offline instance of hishtory with no syncing
IsOffline bool `json:"is_offline"`
}
type CustomColumnDefinition struct {

34
client/lib/lib.go
View File

@ -334,8 +334,16 @@ func shouldSkipHiddenCommand(ctx *context.Context, historyLine string) (bool, er
func Setup(args []string) error {
userSecret := uuid.Must(uuid.NewRandom()).String()
isOffline := false
if len(args) > 2 && args[2] != "" {
userSecret = args[2]
if args[2] == "--offline" {
isOffline = true
} else {
if args[2][0] == '-' {
return fmt.Errorf("refusing to set user secret to %#v since it looks like a flag", args[2])
}
userSecret = args[2]
}
}
fmt.Println("Setting secret hishtory key to " + string(userSecret))
@ -345,6 +353,7 @@ func Setup(args []string) error {
config.IsEnabled = true
config.DeviceId = uuid.Must(uuid.NewRandom()).String()
config.ControlRSearchEnabled = true
config.IsOffline = isOffline
err := hctx.SetConfig(config)
if err != nil {
return fmt.Errorf("failed to persist config to disk: %v", err)
@ -358,6 +367,9 @@ func Setup(args []string) error {
db.Exec("DELETE FROM history_entries")
// Bootstrap from remote date
if config.IsOffline {
return nil
}
_, err = ApiGet("/api/v1/register?user_id=" + data.UserId(userSecret) + "&device_id=" + config.DeviceId)
if err != nil {
return fmt.Errorf("failed to register device with backend: %v", err)
@ -1217,10 +1229,10 @@ func ReliableDbCreate(db *gorm.DB, entry interface{}) error {
return nil
}
}
return err
return fmt.Errorf("unrecoverable sqlite error: %v", err)
}
if err != nil && err.Error() != "database is locked (5) (SQLITE_BUSY)" {
return err
return fmt.Errorf("unrecoverable sqlite error: %v", err)
}
}
return fmt.Errorf("failed to create DB entry even with %d retries: %v", i, err)
@ -1287,6 +1299,9 @@ func Redact(ctx *context.Context, query string, force bool) error {
func deleteOnRemoteInstances(ctx *context.Context, historyEntries []*data.HistoryEntry) error {
config := hctx.GetConf(ctx)
if config.IsOffline {
return nil
}
var deletionRequest shared.DeletionRequest
deletionRequest.SendTime = time.Now()
@ -1308,6 +1323,9 @@ func deleteOnRemoteInstances(ctx *context.Context, historyEntries []*data.Histor
func Reupload(ctx *context.Context) error {
config := hctx.GetConf(ctx)
if config.IsOffline {
return nil
}
entries, err := Search(ctx, hctx.GetDb(ctx), "", 0)
if err != nil {
return fmt.Errorf("failed to reupload due to failed search: %v", err)
@ -1340,6 +1358,9 @@ func chunks[k any](slice []k, chunkSize int) [][]k {
func RetrieveAdditionalEntriesFromRemote(ctx *context.Context) error {
db := hctx.GetDb(ctx)
config := hctx.GetConf(ctx)
if config.IsOffline {
return nil
}
respBody, err := ApiGet("/api/v1/query?device_id=" + config.DeviceId + "&user_id=" + data.UserId(config.UserSecret))
if IsOfflineError(err) {
return nil
@ -1364,7 +1385,9 @@ func RetrieveAdditionalEntriesFromRemote(ctx *context.Context) error {
func ProcessDeletionRequests(ctx *context.Context) error {
config := hctx.GetConf(ctx)
if config.IsOffline {
return nil
}
resp, err := ApiGet("/api/v1/get-deletion-requests?user_id=" + data.UserId(config.UserSecret) + "&device_id=" + config.DeviceId)
if IsOfflineError(err) {
return nil
@ -1391,6 +1414,9 @@ func ProcessDeletionRequests(ctx *context.Context) error {
func GetBanner(ctx *context.Context, gitCommit string) ([]byte, error) {
config := hctx.GetConf(ctx)
if config.IsOffline {
return []byte{}, nil
}
url := "/api/v1/banner?commit_hash=" + gitCommit + "&user_id=" + data.UserId(config.UserSecret) + "&device_id=" + config.DeviceId + "&version=" + Version + "&forced_banner=" + os.Getenv("FORCED_BANNER")
return ApiGet(url)
}

28
client/lib/lib_test.go
View File

@ -32,6 +32,34 @@ func TestSetup(t *testing.T) {
if len(data) < 10 {
t.Fatalf("hishtory secret has unexpected length: %d", len(data))
}
config := hctx.GetConf(hctx.MakeContext())
if config.IsOffline != false {
t.Fatalf("hishtory config should have been offline")
}
}
func TestSetupOffline(t *testing.T) {
defer testutils.BackupAndRestore(t)()
defer testutils.RunTestServer()()
homedir, err := os.UserHomeDir()
testutils.Check(t, err)
if _, err := os.Stat(path.Join(homedir, data.HISHTORY_PATH, data.CONFIG_PATH)); err == nil {
t.Fatalf("hishtory secret file already exists!")
}
testutils.Check(t, Setup([]string{"", "", "--offline"}))
if _, err := os.Stat(path.Join(homedir, data.HISHTORY_PATH, data.CONFIG_PATH)); err != nil {
t.Fatalf("hishtory secret file does not exist after Setup()!")
}
data, err := os.ReadFile(path.Join(homedir, data.HISHTORY_PATH, data.CONFIG_PATH))
testutils.Check(t, err)
if len(data) < 10 {
t.Fatalf("hishtory secret has unexpected length: %d", len(data))
}
config := hctx.GetConf(hctx.MakeContext())
if config.IsOffline != true {
t.Fatalf("hishtory config should have been offline, actual=%#v", string(data))
}
}
func TestBuildHistoryEntry(t *testing.T) {

38
hishtory.go
View File

@ -287,6 +287,9 @@ func printDumpStatus(config hctx.ClientConfig) {
}
func getDumpRequests(config hctx.ClientConfig) ([]*shared.DumpRequest, error) {
if config.IsOffline {
return make([]*shared.DumpRequest, 0), nil
}
resp, err := lib.ApiGet("/api/v1/get-dump-requests?user_id=" + data.UserId(config.UserSecret) + "&device_id=" + config.DeviceId)
if lib.IsOfflineError(err) {
return []*shared.DumpRequest{}, nil
@ -334,6 +337,9 @@ func maybeUploadSkippedHistoryEntries(ctx *context.Context) error {
if !config.HaveMissedUploads {
return nil
}
if config.IsOffline {
return nil
}
// Upload the missing entries
db := hctx.GetDb(ctx)
@ -382,19 +388,21 @@ func saveHistoryEntry(ctx *context.Context) {
lib.CheckFatalError(err)
// Persist it remotely
jsonValue, err := lib.EncryptAndMarshal(config, []*data.HistoryEntry{entry})
lib.CheckFatalError(err)
_, err = lib.ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
if err != nil {
if lib.IsOfflineError(err) {
hctx.GetLogger().Printf("Failed to remotely persist hishtory entry because the device is offline!")
if !config.HaveMissedUploads {
config.HaveMissedUploads = true
config.MissedUploadTimestamp = time.Now().Unix()
lib.CheckFatalError(hctx.SetConfig(config))
if !config.IsOffline {
jsonValue, err := lib.EncryptAndMarshal(config, []*data.HistoryEntry{entry})
lib.CheckFatalError(err)
_, err = lib.ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
if err != nil {
if lib.IsOfflineError(err) {
hctx.GetLogger().Printf("Failed to remotely persist hishtory entry because the device is offline!")
if !config.HaveMissedUploads {
config.HaveMissedUploads = true
config.MissedUploadTimestamp = time.Now().Unix()
lib.CheckFatalError(hctx.SetConfig(config))
}
} else {
lib.CheckFatalError(err)
}
} else {
lib.CheckFatalError(err)
}
}
@ -422,8 +430,10 @@ func saveHistoryEntry(ctx *context.Context) {
reqBody, err := json.Marshal(encEntries)
lib.CheckFatalError(err)
for _, dumpRequest := range dumpRequests {
_, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId+"&source_device_id="+config.DeviceId, "application/json", reqBody)
lib.CheckFatalError(err)
if !config.IsOffline {
_, err := lib.ApiPost("/api/v1/submit-dump?user_id="+dumpRequest.UserId+"&requesting_device_id="+dumpRequest.RequestingDeviceId+"&source_device_id="+config.DeviceId, "application/json", reqBody)
lib.CheckFatalError(err)
}
}
}
}

13
shared/testutils/testutils.go
View File

@ -29,13 +29,7 @@ func ResetLocalState(t *testing.T) {
t.Fatalf("failed to retrieve homedir: %v", err)
}
_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, data.DB_PATH))
_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, DB_WAL_PATH))
_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, data.CONFIG_PATH))
_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, "hishtory"))
_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, "config.sh"))
_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, "config.zsh"))
_ = os.Remove(path.Join(homedir, data.HISHTORY_PATH, "config.fish"))
_ = os.RemoveAll(path.Join(homedir, data.HISHTORY_PATH))
}
func BackupAndRestore(t *testing.T) func() {
@ -43,7 +37,10 @@ func BackupAndRestore(t *testing.T) func() {
}
func getBackPath(file, id string) string {
return strings.Replace(file, data.HISHTORY_PATH, data.HISHTORY_PATH+".test", 1) + id
if strings.Contains(file, "/"+data.HISHTORY_PATH+"/") {
return strings.Replace(file, data.HISHTORY_PATH, data.HISHTORY_PATH+".test", 1) + id
}
return file + ".bak" + id
}
func BackupAndRestoreWithId(t *testing.T, id string) func() {