extern/hishtory
1
0
Fork 0
mirror of https://github.com/ddworken/hishtory.git synced 2024-12-23 23:39:02 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactoring, better tests, commit hash, banner, and tested working locally

Browse Source
This commit is contained in:
David Dworken 2022-04-06 22:43:07 -07:00
parent 684511f4f7
commit 1065fff062
11 changed files with 185 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile
View File

@ -1,8 +1,10 @@
test:
HISHTORY_TEST=1 go test -p 1 ./...
build-static:
go build -o web/landing/www/binaries/hishtory-linux client/client.go
build-binary:
go build -o web/landing/www/binaries/hishtory-linux -ldflags "-X main.GitCommit=`git rev-list -1 HEAD`" client/client.go
build-static: build-binary
docker build -t gcr.io/dworken-k8s/hishtory-static -f web/caddy/Dockerfile .
build-api:
@ -15,3 +17,5 @@ deploy-static: build-static
deploy-api: build-api
docker push gcr.io/dworken-k8s/hishtory-api
kubectl patch deployment hishtory-api -p "{\"spec\":{\"template\":{\"metadata\":{\"labels\":{\"ts\":\"`date|sed -e 's/ /_/g'|sed -e 's/:/-/g'`\"}}}}}}"
deploy: deploy-static deploy-api

48
client/client.go
View File

@ -15,6 +15,8 @@ import (
"github.com/ddworken/hishtory/shared"
)
var GitCommit string = "Unknown"
func main() {
if len(os.Args) == 1 {
fmt.Println("Must specify a command! Do you mean `hishtory query`?")
@ -38,8 +40,12 @@ func main() {
case "status":
config, err := lib.GetConfig()
lib.CheckFatalError(err)
fmt.Print("Hishtory: Offline Mode\nEnabled: ")
fmt.Print("Hishtory: e2e sync\nEnabled: ")
fmt.Print(config.IsEnabled)
fmt.Print("\nSecret Key: ")
fmt.Print(config.UserSecret)
fmt.Print("\nCommit Hash: ")
fmt.Print(GitCommit)
fmt.Print("\n")
case "update":
lib.CheckFatalError(lib.Update("https://hishtory.dev/binaries/hishtory-linux"))
@ -58,6 +64,9 @@ func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error {
return fmt.Errorf("failed to pull latest history entries from the backend: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("failed to retrieve data from backend, status_code=%d", resp.StatusCode)
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to read latest history entries response body: %v", err)
@ -72,21 +81,44 @@ func retrieveAdditionalEntriesFromRemote(db *gorm.DB) error {
if err != nil {
return fmt.Errorf("failed to decrypt history entry from server: %v", err)
}
// TODO: Is this creating duplicate entries?
lib.AddToDbIfNew(db, decEntry)
}
return nil
}
func query(query string) {
db, err := shared.OpenLocalSqliteDb()
db, err := lib.OpenLocalSqliteDb()
lib.CheckFatalError(err)
lib.CheckFatalError(retrieveAdditionalEntriesFromRemote(db))
lib.CheckFatalError(displayBannerIfSet())
data, err := shared.Search(db, query, 25)
lib.CheckFatalError(err)
lib.DisplayResults(data, false)
}
func displayBannerIfSet() error {
config, err := lib.GetConfig()
if err != nil {
return fmt.Errorf("failed to get config: %v", err)
}
url := lib.GetServerHostname() + "/api/v1/banner?commit_hash=" + GitCommit + "&device_id=" + config.DeviceId + "&forced_banner=" + os.Getenv("FORCED_BANNER")
resp, err := http.Get(url)
if err != nil {
return fmt.Errorf("failed to call /api/v1/banner: %v", err)
}
if resp.StatusCode != 200 {
return fmt.Errorf("failed to call %s, status_code=%d", url, resp.StatusCode)
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to read /api/v1/banner response body: %v", err)
}
if len(data) > 0 {
fmt.Printf(string(data))
}
return nil
}
func saveHistoryEntry() {
config, err := lib.GetConfig()
lib.CheckFatalError(err)
@ -97,7 +129,7 @@ func saveHistoryEntry() {
lib.CheckFatalError(err)
// Persist it locally
db, err := shared.OpenLocalSqliteDb()
db, err := lib.OpenLocalSqliteDb()
lib.CheckFatalError(err)
result := db.Create(entry)
lib.CheckFatalError(result.Error)
@ -107,12 +139,15 @@ func saveHistoryEntry() {
lib.CheckFatalError(err)
jsonValue, err := json.Marshal([]shared.EncHistoryEntry{encEntry})
lib.CheckFatalError(err)
_, err = http.Post(lib.GetServerHostname()+"/api/v1/esubmit", "application/json", bytes.NewBuffer(jsonValue))
resp, err := http.Post(lib.GetServerHostname()+"/api/v1/esubmit", "application/json", bytes.NewBuffer(jsonValue))
lib.CheckFatalError(err)
if resp.StatusCode != 200 {
lib.CheckFatalError(fmt.Errorf("failed to submit result to backend, status_code=%d", resp.StatusCode))
}
}
func export() {
db, err := shared.OpenLocalSqliteDb()
db, err := lib.OpenLocalSqliteDb()
lib.CheckFatalError(err)
lib.CheckFatalError(retrieveAdditionalEntriesFromRemote(db))
data, err := shared.Search(db, "", 0)
@ -121,3 +156,4 @@ func export() {
fmt.Println(entry)
}
}

33
client/client_test.go
View File

@ -4,9 +4,11 @@ import (
"bytes"
"io/ioutil"
"os/exec"
"os"
"regexp"
"strings"
"testing"
"fmt"
"github.com/ddworken/hishtory/shared"
)
@ -48,19 +50,13 @@ func TestIntegrationWithNewDevice(t *testing.T) {
gvm use go1.17
cd /home/david/code/hishtory/
go build -o /tmp/client client/client.go
/tmp/client install`)
/tmp/client install ` + userSecret)
match, err := regexp.MatchString(`Setting secret hishtory key to .*`, out)
shared.Check(t, err)
if !match {
t.Fatalf("unexpected output from install: %v", out)
}
// Set the secret key to the previous secret key
out = RunInteractiveBashCommands(t, `hishtory init `+userSecret)
if !strings.Contains(out, "Setting secret hishtory key to "+userSecret) {
t.Fatalf("Failed to re-init with the user secret: %v", out)
}
// Querying should show the history from the previous run
out = RunInteractiveBashCommands(t, "hishtory query")
expected := []string{"echo thisisrecorded", "hishtory enable", "echo bar", "echo foo", "ls /foo", "ls /bar", "ls /a"}
@ -97,6 +93,13 @@ func TestIntegrationWithNewDevice(t *testing.T) {
t.Fatalf("unexpected output from install: %v", out)
}
// Run a command that shouldn't be in the hishtory later on
RunInteractiveBashCommands(t, `echo notinthehistory`)
out = RunInteractiveBashCommands(t, "hishtory query")
if !strings.Contains(out, "echo notinthehistory") {
t.Fatalf("output is missing `echo notinthehistory`")
}
// Set the secret key to the previous secret key
out = RunInteractiveBashCommands(t, `hishtory init `+userSecret)
if !strings.Contains(out, "Setting secret hishtory key to "+userSecret) {
@ -114,6 +117,10 @@ func TestIntegrationWithNewDevice(t *testing.T) {
t.Fatalf("output has %#v in it multiple times! out=%#v", item, out)
}
}
// But not from the previous account
if strings.Contains(out, "notinthehistory") {
t.Fatalf("output contains the unexpected item: notinthehistory")
}
RunInteractiveBashCommands(t, "echo mynewercommand")
out = RunInteractiveBashCommands(t, "hishtory query")
@ -141,14 +148,22 @@ func testIntegration(t *testing.T) string {
}
userSecret := matches[1]
// TODO(ddworken): Test the status subcommand
// Test the status subcommand
out = RunInteractiveBashCommands(t, `
hishtory status
`)
if out != "Hishtory: Offline Mode\nEnabled: true\n" {
if out != fmt.Sprintf("Hishtory: e2e sync\nEnabled: true\nSecret Key: %s\nCommit Hash: Unknown\n", userSecret) {
t.Fatalf("status command has unexpected output: %#v", out)
}
// Test the banner
os.Setenv("FORCED_BANNER", "HELLO_FROM_SERVER")
out = RunInteractiveBashCommands(t, `hishtory query`)
if !strings.Contains(out, "HELLO_FROM_SERVER") {
t.Fatalf("hishtory query didn't show the banner message! out=%#v", out)
}
os.Setenv("FORCED_BANNER", "")
// Test recording commands
out = RunInteractiveBashCommands(t, `
ls /a

52
client/lib/lib.go
View File

@ -21,6 +21,7 @@ import (
"time"
"gorm.io/gorm"
"gorm.io/driver/sqlite"
"github.com/fatih/color"
"github.com/google/uuid"
@ -122,26 +123,40 @@ func Setup(args []string) error {
}
fmt.Println("Setting secret hishtory key to " + string(userSecret))
// Create and set the config
var config ClientConfig
config.UserSecret = userSecret
config.IsEnabled = true
config.DeviceId = uuid.Must(uuid.NewRandom()).String()
err := SetConfig(config)
if err != nil {
return fmt.Errorf("failed to persist config to disk: %v", err)
}
_, err = http.Get(GetServerHostname() + "/api/v1/eregister?user_id=" + shared.UserId(userSecret) + "&device_id=" + config.DeviceId)
// Drop all existing data
db, err := OpenLocalSqliteDb()
if err != nil {
return fmt.Errorf("failed to open DB: %v", err)
}
db.Exec("DELETE FROM history_entries")
// Bootstrap from remote date
resp, err := http.Get(GetServerHostname() + "/api/v1/eregister?user_id=" + shared.UserId(userSecret) + "&device_id=" + config.DeviceId)
if err != nil {
return fmt.Errorf("failed to register device with backend: %v", err)
}
if resp.StatusCode != 200 {
return fmt.Errorf("failed to register device with backend, status_code=%d", resp.StatusCode)
}
resp, err := http.Get(GetServerHostname() + "/api/v1/ebootstrap?user_id=" + shared.UserId(userSecret) + "&device_id=" + config.DeviceId)
resp, err = http.Get(GetServerHostname() + "/api/v1/ebootstrap?user_id=" + shared.UserId(userSecret) + "&device_id=" + config.DeviceId)
if err != nil {
return fmt.Errorf("failed to bootstrap device from the backend: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("failed to bootstrap device with data from existing devices, status_code=%d", resp.StatusCode)
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to read bootstrap response body: %v", err)
@ -151,10 +166,6 @@ func Setup(args []string) error {
if err != nil {
return fmt.Errorf("failed to load JSON response: %v", err)
}
db, err := shared.OpenLocalSqliteDb()
if err != nil {
return fmt.Errorf("failed to open DB: %v", err)
}
for _, entry := range retrievedEntries {
decEntry, err := shared.DecryptHistoryEntry(userSecret, *entry)
if err != nil {
@ -389,7 +400,7 @@ func Update(url string) error {
}
err = syscall.Unlink(path.Join(homedir, shared.HISHTORY_PATH, "hishtory"))
if err != nil {
return fmt.Errorf("Failed to unlink: %v", err)
return fmt.Errorf("Failed to unlink %s: %v", path.Join(homedir, shared.HISHTORY_PATH, "hishtory"), err)
}
cmd = exec.Command("/tmp/hishtory-client", "install")
err = cmd.Run()
@ -405,3 +416,28 @@ func GetServerHostname() string {
}
return "https://api.hishtory.dev"
}
func OpenLocalSqliteDb() (*gorm.DB, error) {
homedir, err := os.UserHomeDir()
if err != nil {
return nil, fmt.Errorf("failed to get user's home directory: %v", err)
}
err = os.MkdirAll(path.Join(homedir, shared.HISHTORY_PATH), 0744)
if err != nil {
return nil, fmt.Errorf("failed to create ~/.hishtory dir: %v", err)
}
db, err := gorm.Open(sqlite.Open(path.Join(homedir, shared.HISHTORY_PATH, shared.DB_PATH)), &gorm.Config{SkipDefaultTransaction: true})
if err != nil {
return nil, fmt.Errorf("failed to connect to the DB: %v", err)
}
tx, err := db.DB()
if err != nil {
return nil, err
}
err = tx.Ping()
if err != nil {
return nil, err
}
db.AutoMigrate(&shared.HistoryEntry{})
return db, nil
}

45
client/lib/lib_test.go
View File

@ -70,3 +70,48 @@ func TestGetUserSecret(t *testing.T) {
t.Fatalf("GetUserSecret() returned the same values for different setups! val=%v", secret1)
}
}
func TestPersist(t *testing.T) {
defer shared.BackupAndRestore(t)()
db, err := OpenLocalSqliteDb()
shared.Check(t, err)
entry := shared.MakeFakeHistoryEntry("ls ~/")
db.Create(entry)
var historyEntries []*shared.HistoryEntry
result := db.Find(&historyEntries)
shared.Check(t, result.Error)
if len(historyEntries) != 1 {
t.Fatalf("DB has %d entries, expected 1!", len(historyEntries))
}
dbEntry := historyEntries[0]
if !shared.EntryEquals(entry, *dbEntry) {
t.Fatalf("DB data is different than input! \ndb =%#v \ninput=%#v", *dbEntry, entry)
}
}
func TestSearch(t *testing.T) {
defer shared.BackupAndRestore(t)()
db, err := OpenLocalSqliteDb()
shared.Check(t, err)
// Insert data
entry1 := shared.MakeFakeHistoryEntry("ls /foo")
db.Create(entry1)
entry2 := shared.MakeFakeHistoryEntry("ls /bar")
db.Create(entry2)
// Search for data
results, err := shared.Search(db, "ls", 5)
shared.Check(t, err)
if len(results) != 2 {
t.Fatalf("Search() returned %d results, expected 2!", len(results))
}
if !shared.EntryEquals(*results[0], entry2) {
t.Fatalf("Search()[0]=%#v, expected: %#v", results[0], entry2)
}
if !shared.EntryEquals(*results[1], entry1) {
t.Fatalf("Search()[0]=%#v, expected: %#v", results[1], entry1)
}
}

19
server/server.go
View File

@ -32,6 +32,7 @@ func apiESubmitHandler(w http.ResponseWriter, r *http.Request) {
panic(fmt.Sprintf("body=%#v, err=%v", data, err))
}
GLOBAL_DB.Where("user_id = ?")
fmt.Printf("apiESubmitHandler: received request containg %d EncHistoryEntry\n", len(entries))
for _, entry := range entries {
tx := GLOBAL_DB.Where("user_id = ?", entry.UserId)
var devices []*shared.Device
@ -42,9 +43,13 @@ func apiESubmitHandler(w http.ResponseWriter, r *http.Request) {
if len(devices) == 0 {
panic(fmt.Errorf("Found no devices associated with user_id=%s, can't save history entry!", entry.UserId))
}
fmt.Printf("apiESubmitHandler: Found %d devices\n", len(devices))
for _, device := range devices {
entry.DeviceId = device.DeviceId
GLOBAL_DB.Create(&entry)
result := GLOBAL_DB.Create(&entry)
if result.Error != nil {
panic(result.Error)
}
}
}
}
@ -61,6 +66,7 @@ func apiEQueryHandler(w http.ResponseWriter, r *http.Request) {
if result.Error != nil {
panic(fmt.Errorf("DB query error: %v", result.Error))
}
fmt.Printf("apiEQueryHandler: Found %d entries\n", len(historyEntries))
resp, err := json.Marshal(historyEntries)
if err != nil {
panic(err)
@ -90,13 +96,20 @@ func apiERegisterHandler(w http.ResponseWriter, r *http.Request) {
GLOBAL_DB.Create(&shared.Device{UserId: userId, DeviceId: deviceId})
}
func apiBannerHandler(w http.ResponseWriter, r *http.Request) {
commitHash := r.URL.Query().Get("commit_hash")
deviceId := r.URL.Query().Get("device_id")
forcedBanner := r.URL.Query().Get("forced_banner")
fmt.Printf("apiBannerHandler: commit_hash=%#v, device_id=%#v, forced_banner=%#v\n", commitHash, deviceId, forcedBanner)
w.Write([]byte(forcedBanner))
}
func OpenDB() (*gorm.DB, error) {
if shared.IsTestEnvironment() {
db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
if err != nil {
return nil, fmt.Errorf("failed to connect to the DB: %v", err)
}
db.AutoMigrate(&shared.HistoryEntry{})
db.AutoMigrate(&shared.EncHistoryEntry{})
db.AutoMigrate(&shared.Device{})
return db, nil
@ -106,7 +119,6 @@ func OpenDB() (*gorm.DB, error) {
if err != nil {
return nil, fmt.Errorf("failed to connect to the DB: %v", err)
}
db.AutoMigrate(&shared.HistoryEntry{})
db.AutoMigrate(&shared.EncHistoryEntry{})
db.AutoMigrate(&shared.Device{})
return db, nil
@ -138,5 +150,6 @@ func main() {
http.HandleFunc("/api/v1/equery", apiEQueryHandler)
http.HandleFunc("/api/v1/ebootstrap", apiEBootstrapHandler)
http.HandleFunc("/api/v1/eregister", apiERegisterHandler)
http.HandleFunc("/api/v1/banner", apiBannerHandler)
log.Fatal(http.ListenAndServe(":8080", nil))
}

1
server/server_test.go
View File

@ -16,7 +16,6 @@ func TestESubmitThenQuery(t *testing.T) {
// Set up
defer shared.BackupAndRestore(t)()
InitDB()
// shared.Check(t, shared.Setup([]string{}))
// Register a few devices
userId := shared.UserId("key")

29
shared/data.go
View File

@ -4,7 +4,6 @@ import (
"fmt"
"io"
"os"
"path"
"strings"
"time"
@ -16,7 +15,6 @@ import (
"encoding/base64"
"encoding/json"
"github.com/google/uuid"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
@ -164,33 +162,6 @@ func IsTestEnvironment() bool {
return os.Getenv("HISHTORY_TEST") != ""
}
func OpenLocalSqliteDb() (*gorm.DB, error) {
homedir, err := os.UserHomeDir()
if err != nil {
return nil, fmt.Errorf("failed to get user's home directory: %v", err)
}
err = os.MkdirAll(path.Join(homedir, HISHTORY_PATH), 0744)
if err != nil {
return nil, fmt.Errorf("failed to create ~/.hishtory dir: %v", err)
}
db, err := gorm.Open(sqlite.Open(path.Join(homedir, HISHTORY_PATH, DB_PATH)), &gorm.Config{SkipDefaultTransaction: true})
if err != nil {
return nil, fmt.Errorf("failed to connect to the DB: %v", err)
}
tx, err := db.DB()
if err != nil {
return nil, err
}
err = tx.Ping()
if err != nil {
return nil, err
}
db.AutoMigrate(&HistoryEntry{})
db.AutoMigrate(&EncHistoryEntry{})
db.AutoMigrate(&Device{})
return db, nil
}
func Search(db *gorm.DB, query string, limit int) ([]*HistoryEntry, error) {
tokens, err := tokenize(query)
if err != nil {

46
shared/data_test.go
View File

@ -4,52 +4,6 @@ import (
"testing"
)
func TestPersist(t *testing.T) {
defer BackupAndRestore(t)()
// Check(t, Setup([]string{}))
db, err := OpenLocalSqliteDb()
Check(t, err)
entry := MakeFakeHistoryEntry("ls ~/")
db.Create(entry)
var historyEntries []*HistoryEntry
result := db.Find(&historyEntries)
Check(t, result.Error)
if len(historyEntries) != 1 {
t.Fatalf("DB has %d entries, expected 1!", len(historyEntries))
}
dbEntry := historyEntries[0]
if !EntryEquals(entry, *dbEntry) {
t.Fatalf("DB data is different than input! \ndb =%#v \ninput=%#v", *dbEntry, entry)
}
}
func TestSearch(t *testing.T) {
defer BackupAndRestore(t)()
// Check(t, Setup([]string{}))
db, err := OpenLocalSqliteDb()
Check(t, err)
// Insert data
entry1 := MakeFakeHistoryEntry("ls /foo")
db.Create(entry1)
entry2 := MakeFakeHistoryEntry("ls /bar")
db.Create(entry2)
// Search for data
results, err := Search(db, "ls", 5)
Check(t, err)
if len(results) != 2 {
t.Fatalf("Search() returned %d results, expected 2!", len(results))
}
if !EntryEquals(*results[0], entry2) {
t.Fatalf("Search()[0]=%#v, expected: %#v", results[0], entry2)
}
if !EntryEquals(*results[1], entry1) {
t.Fatalf("Search()[0]=%#v, expected: %#v", results[1], entry1)
}
}
func TestEncryptDecrypt(t *testing.T) {
k1, err := EncryptionKey("key")
Check(t, err)

BIN
web/landing/www/binaries/hishtory-linux
View File

Binary file not shown.

15
web/landing/www/index.html
View File

@ -103,14 +103,15 @@
</li>
</ul>
<div class="tab-content">
<div class="tab-pane active" id="install-online" role="tabpanel" aria-labelledby="home-tab">
<br>To install hishtory in online mode on your first machine:
<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; <code>curl -o hishtory https://hishtory.dev/hishtory-online; chmod +x hishtory; ./hishtory install</code>
<br> To install hishtory in online mode on your other machines:
<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; <code>curl -o hishtory https://hishtory.dev/hishtory-online; chmod +x hishtory; ./hishtory install $YOUR_HISHTORY_SECRET</code>
</div>
<div class="tab-pane" id="install-offline" role="tabpanel" aria-labelledby="profile-tab"><br> To install hishtory in offline mode: <br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; <code>curl -o hishtory https://hishtory.dev/hishtory-offline; chmod +x hishtory; ./hishtory install</code>
<div class="tab-pane active" id="install-first" role="tabpanel" aria-labelledby="home-tab">
<br>To install hishtory on your first machine:
<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; <code>curl -o hishtory https://hishtory.dev/binaries/hishtory-linux; chmod +x hishtory; ./hishtory install</code>
</div>
<div class="tab-pane" id="install-offline" role="tabpanel" aria-labelledby="profile-tab">
<br>To install hishtory on your second machine, you must first retrieve your secret key from first first machine. To do so, run <code>hishtory status</code> and copy your "Secret Key".
<br> Then to install it on your second machine:
<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; <code>curl -o hishtory https://hishtory.dev/binaries/hishtory-linux; chmod +x hishtory; ./hishtory install $YOUR_HISHTORY_SECRET</code>
</div>
</div>
</div>
</p>