Add support for fully offline binary via compile-time 'offline' tag (#272)

* Add support for fully offline binary via compile-time 'offline' tag

* Update docs

README.md

@@ -165,6 +165,8 @@ curl https://hishtory.dev/install.py | python3 - --offline
 
 This disables syncing completely so that the client will not rely on the hiSHtory backend at all. You can also change the syncing status via `hishtory syncing enable` or `hishtory syncing disable`.
 
+For more information on offline mode, see [here](https://github.com/ddworken/hishtory/blob/master/docs/offline-binary.md).
+
 </blockquote></details>
 
 <details>

client/client_test.go

@@ -54,6 +54,15 @@ func TestMain(m *testing.M) {
 		panic(fmt.Sprintf("failed to build client: %v", err))
 	}
 
+	// Build the fully offline client so it is available in /tmp/client-offline
+	cmd = exec.Command("go", "build", "-o", "/tmp/client-offline", "-tags", "offline")
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
+	err = cmd.Run()
+	if err != nil {
+		panic(fmt.Sprintf("failed to build offline client: %v", err))
+	}
+
 	// Start the tests
 	m.Run()
 }
@@ -3470,4 +3479,58 @@ func TestImportJson(t *testing.T) {
 	testutils.CompareGoldens(t, out, "TestExportJson")
 }
 
+func TestOfflineClient(t *testing.T) {
+	markTestForSharding(t, 21)
+	defer testutils.BackupAndRestore(t)()
+	tester := zshTester{}
+
+	// Install the offline client
+	out := tester.RunInteractiveShell(t, ` /tmp/client-offline install `)
+	r := regexp.MustCompile(`Setting secret hishtory key to (.*)`)
+	matches := r.FindStringSubmatch(out)
+	if len(matches) != 2 {
+		t.Fatalf("Failed to extract userSecret from output=%#v: matches=%#v", out, matches)
+	}
+	assertOnlineStatus(t, Offline)
+
+	// Disable recording so that all our testing commands don't get recorded
+	_, _ = tester.RunInteractiveShellRelaxed(t, ` hishtory disable`)
+	_, _ = tester.RunInteractiveShellRelaxed(t, `hishtory config-set enable-control-r true`)
+	tester.RunInteractiveShell(t, ` HISHTORY_REDACT_FORCE=true hishtory redact set emo pipefail`)
+
+	// Insert a few hishtory entries that we'll use for testing into an empty DB
+	db := hctx.GetDb(hctx.MakeContext())
+	require.NoError(t, db.Where("true").Delete(&data.HistoryEntry{}).Error)
+	e1 := testutils.MakeFakeHistoryEntry("ls ~/")
+	e1.CurrentWorkingDirectory = "/etc/"
+	e1.Hostname = "server"
+	e1.ExitCode = 127
+	require.NoError(t, db.Create(e1).Error)
+	require.NoError(t, db.Create(testutils.MakeFakeHistoryEntry("ls ~/foo/")).Error)
+	require.NoError(t, db.Create(testutils.MakeFakeHistoryEntry("ls ~/bar/")).Error)
+	require.NoError(t, db.Create(testutils.MakeFakeHistoryEntry("echo 'aaaaaa bbbb'")).Error)
+	require.NoError(t, db.Create(testutils.MakeFakeHistoryEntry("echo 'bar' &")).Error)
+
+	// Check that they're there (and there aren't any other entries)
+	var historyEntries []*data.HistoryEntry
+	db.Model(&data.HistoryEntry{}).Find(&historyEntries)
+	if len(historyEntries) != 5 {
+		t.Fatalf("expected to find 6 history entries, actual found %d: %#v", len(historyEntries), historyEntries)
+	}
+	out = tester.RunInteractiveShell(t, `hishtory export`)
+	testutils.CompareGoldens(t, out, "testControlR-InitialExport")
+
+	// And check that the control-r binding brings up the search
+	out = captureTerminalOutputWithShellName(t, tester, tester.ShellName(), []string{"C-R"})
+	split := strings.Split(out, "\n\n\n")
+	out = strings.TrimSpace(split[len(split)-1])
+	testutils.CompareGoldens(t, out, "testControlR-Initial")
+
+	// And check that even if syncing is enabled, the fully offline client will never send an HTTP request
+	out, err := tester.RunInteractiveShellRelaxed(t, `hishtory syncing enable`)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "panic: Cannot GetHttpClient() from a hishtory client compiled with the offline tag!")
+}
+
+
 // TODO: somehow test/confirm that hishtory works even if only bash/only zsh is installed

client/cmd/install.go

@@ -49,7 +49,7 @@ var installCmd = &cobra.Command{
 		if strings.HasPrefix(secretKey, "-") {
 			lib.CheckFatalError(fmt.Errorf("secret key %#v looks like a CLI flag, please use a secret key that does not start with a -", secretKey))
 		}
-		lib.CheckFatalError(install(secretKey, *offlineInstall, *skipConfigModification || *skipUpdateConfigModification))
+		lib.CheckFatalError(install(secretKey, *offlineInstall || lib.IsOfflineBinary(), *skipConfigModification || *skipUpdateConfigModification))
 		if os.Getenv("HISHTORY_SKIP_INIT_IMPORT") == "" {
 			db, err := hctx.OpenLocalSqliteDb()
 			lib.CheckFatalError(err)

client/cmd/redact.go

@@ -28,7 +28,7 @@ var redactCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		ctx := hctx.MakeContext()
 		skipOnlineRedaction := false
-		if !lib.CanReachHishtoryServer(ctx) {
+		if !hctx.GetConf(ctx).IsOffline && !lib.CanReachHishtoryServer(ctx) {
 			fmt.Printf("Cannot reach hishtory backend (is this device offline?) so redaction will only apply to this device and not other synced devices. Would you like to continue with a local-only redaction anyways? [y/N] ")
 			reader := bufio.NewReader(os.Stdin)
 			resp, err := reader.ReadString('\n')

client/cmd/syncing.go

@@ -14,6 +14,7 @@ import (
 var syncingCmd = &cobra.Command{
 	Use:       "syncing",
 	Short:     "Configure syncing to enable or disable syncing with the hishtory backend",
+	Long:      "Run `hishtory syncing disable` to disable syncing and `hishtory syncing enable` to enable syncing.",
 	ValidArgs: []string{"disable", "enable"},
 	Args:      cobra.MatchAll(cobra.OnlyValidArgs, cobra.ExactArgs(1)),
 	Run: func(cmd *cobra.Command, args []string) {

client/cmd/update.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"os/exec"
 	"path"
@@ -287,7 +286,7 @@ func downloadFile(filename, url string) error {
 	}
 
 	// Download the data
-	resp, err := http.Get(url)
+	resp, err := lib.GetHttpClient().Get(url)
 	if err != nil {
 		return fmt.Errorf("failed to download file at %s to %s: %w", url, filename, err)
 	}

client/lib/lib.go

@@ -460,7 +460,7 @@ func ApiGet(ctx context.Context, path string) ([]byte, error) {
 	req.Header.Set("X-Hishtory-Version", "v0."+Version)
 	req.Header.Set("X-Hishtory-Device-Id", hctx.GetConf(ctx).DeviceId)
 	req.Header.Set("X-Hishtory-User-Id", data.UserId(hctx.GetConf(ctx).UserSecret))
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := GetHttpClient().Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to GET %s%s: %w", GetServerHostname(), path, err)
 	}
@@ -490,7 +490,7 @@ func ApiPost(ctx context.Context, path, contentType string, reqBody []byte) ([]b
 	req.Header.Set("X-Hishtory-Version", "v0."+Version)
 	req.Header.Set("X-Hishtory-Device-Id", hctx.GetConf(ctx).DeviceId)
 	req.Header.Set("X-Hishtory-User-Id", data.UserId(hctx.GetConf(ctx).UserSecret))
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := GetHttpClient().Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to POST %s: %w", GetServerHostname()+path, err)
 	}

client/lib/net.go

@@ -0,0 +1,16 @@
+//go:build !offline
+// +build !offline
+
+package lib
+
+import (
+	"net/http"
+)
+
+func GetHttpClient() *http.Client {
+	return http.DefaultClient
+}
+
+func IsOfflineBinary() bool {
+	return false
+}

client/lib/net_disabled.go

@@ -0,0 +1,14 @@
+//go:build offline
+// +build offline
+
+package lib
+
+import "net/http"
+
+func GetHttpClient() *http.Client {
+	panic("Cannot GetHttpClient() from a hishtory client compiled with the offline tag!")
+}
+
+func IsOfflineBinary() bool {
+	return true
+}

docs/offline-binary.md

@@ -0,0 +1,14 @@
+# Offline Binary
+
+hiSHtory supports disabling syncing at install-time via `curl https://hishtory.dev/install.py | python3 - --offline` or at config-time via `hishtory syncing disable`. This will disable persisting your (encrypted) history on the backend API server. For most users, this is the recommended option for running hiSHtory in an offline environment since it still supports opt-in updates via `hishtory update`.
+
+But, if you need stronger guarantees that hiSHtory will not make any network requests, this can also be done by compiling your own copy of hiSHtory with the `offline` tag. This will statically link in `net_disabled.go` which will guarantee that the binary cannot make any HTTP requests. To use this:
+
+```
+git clone https://github.com/ddworken/hishtory
+cd hishtory
+go build -tags offline
+./hishtory install
+```
+
+This binary will be entirely offline and is guaranteed to never make any requests to `api.hishtory.dev`.

shared/ai/ai.go

@@ -10,6 +10,7 @@ import (
 	"strconv"
 
 	"github.com/ddworken/hishtory/client/hctx"
+	"github.com/ddworken/hishtory/client/lib"
 
 	"golang.org/x/exp/slices"
 )
@@ -76,7 +77,7 @@ func GetAiSuggestionsViaOpenAiApi(apiEndpoint, query, shellName, osName, overrid
 	if apiKey != "" {
 		req.Header.Set("Authorization", "Bearer "+apiKey)
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := lib.GetHttpClient().Do(req)
 	if err != nil {
 		return nil, OpenAiUsage{}, fmt.Errorf("failed to query OpenAI API: %w", err)
 	}