tests are passing and getting close now. Need to test the live update flow along with more thorough testing for everything

client/client.go

@@ -5,9 +5,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"io/ioutil"
 	"os"
 	"strings"
 
+	"gorm.io/gorm"
+
 	"github.com/ddworken/hishtory/shared"
 	"github.com/ddworken/hishtory/client/lib"
 )
@@ -45,9 +48,40 @@ func main() {
 	}
 }
 
+func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error {
+	config, err := lib.GetConfig()
+	if err != nil {
+		return err 
+	}
+	resp, err := http.Get(lib.GetServerHostname()+"/api/v1/equery?device_id=" + config.DeviceId)
+	if err != nil {
+		return fmt.Errorf("failed to pull latest history entries from the backend: %v", err)
+	}
+	defer resp.Body.Close()
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to read latest history entries response body: %v", err)
+	}
+	var retrievedEntries []*shared.EncHistoryEntry
+	err = json.Unmarshal(data, &retrievedEntries)
+	if err != nil {
+		return fmt.Errorf("failed to load JSON response: %v", err)
+	}
+	for _, entry := range retrievedEntries {
+		decEntry, err := shared.DecryptHistoryEntry(config.UserSecret, *entry)
+		if err != nil {
+			return fmt.Errorf("failed to decrypt history entry from server: %v", err)
+		}
+		// TODO: Is this creating duplicate entries?
+		lib.AddToDbIfNew(db, decEntry)
+	}
+	return nil 
+}
+
 func query(query string) {
 	db, err := shared.OpenLocalSqliteDb()
 	lib.CheckFatalError(err)
+	lib.CheckFatalError(retrieveAdditionalEntriesFromRemote(db))
 	data, err := shared.Search(db, query, 25)
 	lib.CheckFatalError(err)
 	lib.DisplayResults(data, false)
@@ -71,7 +105,7 @@ func saveHistoryEntry() {
 	// Persist it remotely
 	encEntry, err := shared.EncryptHistoryEntry(config.UserSecret, *entry)
 	lib.CheckFatalError(err)
-	jsonValue, err := json.Marshal(encEntry)
+	jsonValue, err := json.Marshal([]shared.EncHistoryEntry{encEntry})
 	lib.CheckFatalError(err)
 	_, err = http.Post(lib.GetServerHostname()+"/api/v1/esubmit", "application/json", bytes.NewBuffer(jsonValue))
 	lib.CheckFatalError(err)
@@ -80,6 +114,7 @@ func saveHistoryEntry() {
 func export() {
 	db, err := shared.OpenLocalSqliteDb()
 	lib.CheckFatalError(err)
+	lib.CheckFatalError(retrieveAdditionalEntriesFromRemote(db))
 	data, err := shared.Search(db, "", 0)
 	lib.CheckFatalError(err)
 	for _, entry := range data {

client/client_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func RunInteractiveBashCommands(t *testing.T, script string) string {
-	shared.Check(t, ioutil.WriteFile("/tmp/hishtory-test-in.sh", []byte(script), 0600))
+	shared.Check(t, ioutil.WriteFile("/tmp/hishtory-test-in.sh", []byte("set -euo pipefail\n" + script), 0600))
 	cmd := exec.Command("bash", "-i")
 	cmd.Stdin = strings.NewReader(script)
 	var out bytes.Buffer
@@ -26,14 +26,28 @@ func RunInteractiveBashCommands(t *testing.T, script string) string {
 func TestIntegration(t *testing.T) {
 	// Set up
 	defer shared.BackupAndRestore(t)()
+	defer shared.RunTestServer(t)()
 
-	// TODO(ddworken): Test status
+	// Run the test 
+	testIntegration(t) 
+}
 
-	// Test install
+func TestIntegrationWithNewDevice(t *testing.T) {
+	// Set up
+	defer shared.BackupAndRestore(t)()
+	defer shared.RunTestServer(t)()
+
+	// Run the test 
+	userSecret := testIntegration(t) 
+
+	// Clear all local state
+	shared.ResetLocalState(t)
+
+	// Install it again
 	out := RunInteractiveBashCommands(t, `
 	gvm use go1.17
-	cd ../../
-	go build -o /tmp/client clients/local/client.go
+	cd /home/david/code/hishtory/
+	go build -o /tmp/client client/client.go
 	/tmp/client install`)
 	match, err := regexp.MatchString(`Setting secret hishtory key to .*`, out)
 	shared.Check(t, err)
@@ -41,6 +55,55 @@ func TestIntegration(t *testing.T) {
 		t.Fatalf("unexpected output from install: %v", out)
 	}
 
+	// Set the secret key to the previous secret key 
+	out = RunInteractiveBashCommands(t, `hishtory init ` + userSecret)
+	if !strings.Contains(out, "Setting secret hishtory key to " + userSecret) {
+		t.Fatalf("Failed to re-init with the user secret: %v", out)
+	}
+
+	// Querying should show the history from the previous run 
+	out = RunInteractiveBashCommands(t, "hishtory query")
+	expected := []string{"echo thisisrecorded", "hishtory enable", "echo bar", "echo foo", "ls /foo", "ls /bar", "ls /a"}
+	for _, item := range expected {
+		if !strings.Contains(out, item) {
+			t.Fatalf("output is missing expected item %#v: %#v", item, out)
+		}
+		if strings.Count(out, item) != 1 {
+			t.Fatalf("output has %#v in it multiple times! out=%#v", item, out)
+		}
+	}
+
+	RunInteractiveBashCommands(t, "echo mynewcommand")
+	out = RunInteractiveBashCommands(t, "hishtory query")
+	if !strings.Contains(out, "echo mynewcommand") {
+		t.Fatalf("output is missing `echo mynewcommand`")
+	}
+	if strings.Count(out, "echo mynewcommand") != 1 {
+		t.Fatalf("output has `echo mynewcommand` the wrong number of times")
+	}
+
+	// TODO: Set up a third client and check it gets commands from both previous ones
+
+	// TODO: Test the live update flow
+}
+
+func testIntegration(t *testing.T) string {
+	// TODO(ddworken): Test the status subcommand
+
+	// Test install
+	out := RunInteractiveBashCommands(t, `
+	gvm use go1.17
+	cd /home/david/code/hishtory
+	go build -o /tmp/client client/client.go
+	/tmp/client install`)
+	r := regexp.MustCompile(`Setting secret hishtory key to (.*)`)
+	matches := r.FindStringSubmatch(out)
+	if len(matches) != 2 {
+		t.Fatalf("Failed to extract userSecret from output: matches=%#v", matches)
+	}
+	userSecret := matches[1]
+	
+
 	// Test recording commands
 	out = RunInteractiveBashCommands(t, `
 		ls /a
@@ -79,12 +142,17 @@ func TestIntegration(t *testing.T) {
 		if !strings.Contains(out, item) {
 			t.Fatalf("output is missing expected item %#v: %#v", item, out)
 		}
+		if strings.Count(out, item) != 1 {
+			t.Fatalf("output has %#v in it multiple times! out=%#v", item, out)
+		}
 	}
 	for _, item := range unexpected {
 		if strings.Contains(out, item) {
 			t.Fatalf("output is containing unexpected item %#v: %#v", item, out)
 		}
 	}
+
+	return userSecret
 }
 
 // TODO(ddworken): Test export

client/lib/lib.go

@@ -19,6 +19,8 @@ import (
 	"time"
 	"net/http"
 
+	"gorm.io/gorm"
+
 	"github.com/fatih/color"
 	"github.com/google/uuid"
 	"github.com/rodaine/table"
@@ -96,7 +98,7 @@ func BuildHistoryEntry(args []string) (*shared.HistoryEntry, error) {
 		return nil, fmt.Errorf("failed to build history entry: %v", err)
 	}
 	entry.Hostname = hostname
-	
+
 	return &entry, nil
 }
 
@@ -133,9 +135,51 @@ func Setup(args []string) error {
 	if err != nil {
 		return fmt.Errorf("failed to register device with backend: %v", err)
 	}
+
+	resp, err := http.Get(GetServerHostname()+"/api/v1/ebootstrap?user_id=" + shared.UserId(userSecret) + "&device_id=" + config.DeviceId)
+	if err != nil {
+		return fmt.Errorf("failed to bootstrap device from the backend: %v", err)
+	}
+	defer resp.Body.Close()
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to read bootstrap response body: %v", err)
+	}
+	var retrievedEntries []*shared.EncHistoryEntry
+	err = json.Unmarshal(data, &retrievedEntries)
+	if err != nil {
+		return fmt.Errorf("failed to load JSON response: %v", err)
+	}
+	db, err := shared.OpenLocalSqliteDb()
+	if err != nil {
+		return fmt.Errorf("failed to open DB: %v", err)
+	}
+	for _, entry := range retrievedEntries {
+		decEntry, err := shared.DecryptHistoryEntry(userSecret, *entry)
+		if err != nil {
+			return fmt.Errorf("failed to decrypt history entry from server: %v", err)
+		}
+		AddToDbIfNew(db, decEntry)
+	}
+
 	return nil 
 }
 
+func AddToDbIfNew(db *gorm.DB, entry shared.HistoryEntry) {
+	tx := db.Where("local_username = ?", entry.LocalUsername)
+	tx = tx.Where("hostname = ?", entry.Hostname)
+	tx = tx.Where("command = ?", entry.Command)
+	tx = tx.Where("current_working_directory = ?", entry.CurrentWorkingDirectory)
+	tx = tx.Where("exit_code = ?", entry.ExitCode)
+	tx = tx.Where("start_time = ?", entry.StartTime)
+	tx = tx.Where("end_time = ?", entry.EndTime)
+	var results []shared.HistoryEntry 
+	tx.Limit(1).Find(&results)
+	if len(results) == 0 {
+		db.Create(entry)
+	}
+}
+
 func DisplayResults(results []*shared.HistoryEntry, displayHostname bool) {
 	headerFmt := color.New(color.FgGreen, color.Underline).SprintfFunc()
 	tbl := table.New("CWD", "Timestamp", "Runtime", "Exit Code", "Command")
@@ -346,5 +390,5 @@ func GetServerHostname() string {
 	if server := os.Getenv("HISHTORY_SERVER"); server != "" {
 		return server
 	}
-	return "http://localhost:8080"
+	return "https://api.hishtory.dev"
 }

client/lib/lib_test.go

@@ -12,6 +12,7 @@ import (
 
 func TestSetup(t *testing.T) {
 	defer shared.BackupAndRestore(t)()
+	defer shared.RunTestServer(t)()
 	homedir, err := os.UserHomeDir()
 	shared.Check(t, err)
 	if _, err := os.Stat(path.Join(homedir, shared.HISHTORY_PATH, shared.CONFIG_PATH)); err == nil {
@@ -30,6 +31,7 @@ func TestSetup(t *testing.T) {
 
 func TestBuildHistoryEntry(t *testing.T) {
 	defer shared.BackupAndRestore(t)()
+	defer shared.RunTestServer(t)()
 	shared.Check(t, Setup([]string{}))
 	entry, err := BuildHistoryEntry([]string{"unused", "saveHistoryEntry", "120", " 123  ls /  ", "1641774958326745663"})
 	shared.Check(t, err)
@@ -52,6 +54,7 @@ func TestBuildHistoryEntry(t *testing.T) {
 
 func TestGetUserSecret(t *testing.T) {
 	defer shared.BackupAndRestore(t)()
+	defer shared.RunTestServer(t)()
 	shared.Check(t, Setup([]string{}))
 	secret1, err := GetUserSecret()
 	shared.Check(t, err)

server/server.go

@@ -4,11 +4,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"io/ioutil"
 	"net/http"
 
 	"github.com/ddworken/hishtory/shared"
 	_ "github.com/lib/pq"
 	"gorm.io/driver/postgres"
+
+	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 )
 
@@ -19,12 +22,15 @@ const (
 var GLOBAL_DB *gorm.DB
 
 func apiESubmitHandler(w http.ResponseWriter, r *http.Request) {
-	decoder := json.NewDecoder(r.Body)
-	var entries []shared.EncHistoryEntry
-	err := decoder.Decode(&entries)
+	data, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		panic(err)
 	}
+	var entries []shared.EncHistoryEntry
+	err = json.Unmarshal(data, &entries)
+	if err != nil {
+		panic(fmt.Sprintf("body=%#v, err=%v", data, err))
+	}
 	GLOBAL_DB.Where("user_id = ?")
 	for _, entry := range entries {
 		tx := GLOBAL_DB.Where("user_id = ?", entry.UserId)
@@ -62,6 +68,7 @@ func apiEQueryHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write(resp)
 }
 
+// TODO: bootstrap is a janky solution for the initial version of this. Long term, need to support deleting entries from the DB which means replacing bootstrap with a queued message sent to any live instances.
 func apiEBootstrapHandler(w http.ResponseWriter, r *http.Request) {
 	userId := r.URL.Query().Get("user_id")
 	tx := GLOBAL_DB.Where("user_id = ?", userId)
@@ -85,7 +92,14 @@ func apiERegisterHandler(w http.ResponseWriter, r *http.Request) {
 
 func OpenDB() (*gorm.DB, error) {
 	if shared.IsTestEnvironment() {
-		return shared.OpenLocalSqliteDb()
+		db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
+		if err != nil {
+			return nil, fmt.Errorf("failed to connect to the DB: %v", err)
+		}
+		db.AutoMigrate(&shared.HistoryEntry{})
+		db.AutoMigrate(&shared.EncHistoryEntry{})
+		db.AutoMigrate(&shared.Device{})
+		return db, nil
 	}
 
 	db, err := gorm.Open(postgres.Open(POSTGRES_DB), &gorm.Config{})

shared/data.go

@@ -21,13 +21,13 @@ import (
 )
 
 type HistoryEntry struct {
-	LocalUsername           string    `json:"local_username"`
-	Hostname                string    `json:"hostname"`
-	Command                 string    `json:"command"`
-	CurrentWorkingDirectory string    `json:"current_working_directory"`
-	ExitCode                int       `json:"exit_code"`
-	StartTime               time.Time `json:"start_time"`
-	EndTime                 time.Time `json:"end_time"`
+	LocalUsername           string    `json:"local_username" gorm:"uniqueIndex:compositeindex"`
+	Hostname                string    `json:"hostname" gorm:"uniqueIndex:compositeindex"`
+	Command                 string    `json:"command" gorm:"uniqueIndex:compositeindex"`
+	CurrentWorkingDirectory string    `json:"current_working_directory" gorm:"uniqueIndex:compositeindex"`
+	ExitCode                int       `json:"exit_code" gorm:"uniqueIndex:compositeindex"`
+	StartTime               time.Time `json:"start_time" gorm:"uniqueIndex:compositeindex"`
+	EndTime                 time.Time `json:"end_time" gorm:"uniqueIndex:compositeindex"`
 }
 
 type EncHistoryEntry struct {

shared/testutils.go

@@ -5,8 +5,21 @@ import (
 	"time"
 	"os"
 	"path"
+	"os/exec"
+	"bytes"
+	"fmt"
 )
 
+func ResetLocalState(t *testing.T) {
+	homedir, err := os.UserHomeDir()
+	if err != nil {
+		t.Fatalf("failed to retrieve homedir: %v", err)
+	}
+	
+	os.Remove(path.Join(homedir, HISHTORY_PATH, DB_PATH))
+	os.Remove(path.Join(homedir, HISHTORY_PATH, CONFIG_PATH))
+}
+
 func BackupAndRestore(t *testing.T) func() {
 	homedir, err := os.UserHomeDir()
 	if err != nil {
@@ -21,6 +34,52 @@ func BackupAndRestore(t *testing.T) func() {
 	}
 }
 
+func buildServer(t *testing.T) {
+	err := os.Chdir("/home/david/code/hishtory/")
+	if err != nil {
+		t.Fatalf("failed to chdir: %v", err)
+	}
+	cmd := exec.Command("go", "build", "-o", "/tmp/server","server/server.go")	
+	var stdout bytes.Buffer
+	cmd.Stdout = &stdout
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	err = cmd.Start()
+	if err != nil {
+		t.Fatalf("failed to start to build server: %v, stderr=%#v, stdout=%#v", err, stderr.String(), stdout.String())
+	}
+	err = cmd.Wait()
+	if err != nil {
+		t.Fatalf("failed to build server: %v, stderr=%#v, stdout=%#v", err, stderr.String(), stdout.String())
+	}
+}
+
+func RunTestServer(t *testing.T) func() {
+	os.Setenv("HISHTORY_SERVER" ,"http://localhost:8080")
+	buildServer(t)
+	cmd := exec.Command( "/tmp/server" )
+		var stdout bytes.Buffer
+	cmd.Stdout = &stdout
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+    err := cmd.Start()
+    if err != nil {
+        t.Fatalf("failed to start server: %v", err)
+    }
+	time.Sleep(time.Second*3)
+    go func() { 
+        cmd.Wait()
+    }()
+	return func() {
+		err := cmd.Process.Kill()
+		if err != nil {
+			t.Fatalf("failed to kill process: %v", err)
+		}
+		fmt.Println(fmt.Sprintf("stderr=%#v, stdout=%#v", stderr.String(), stdout.String()))
+	}
+
+}
+
 func Check(t *testing.T, err error) {
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)