From 68ed9f2d5db14dc97841b3ae5a76ecfca93a6848 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Thu, 28 Sep 2023 21:49:37 -0700 Subject: [PATCH] Revert all commits since v0.223 to enable me to release a patch on top of v0.223 --- backend/server/internal/database/db.go | 102 ++---------- backend/server/internal/server/server_test.go | 100 ++++++------ backend/server/internal/server/srv.go | 3 +- backend/server/server.go | 22 +-- client/client_test.go | 152 ++++++------------ client/cmd/saveHistoryEntry.go | 75 ++++----- client/cmd/saveHistoryEntry_test.go | 25 ++- client/fuzz_test.go | 9 +- client/lib/config.fish | 4 +- client/lib/config.sh | 7 +- .../testRemoveDuplicateRows-enabled-tquery | 4 + client/lib/lib.go | 26 +-- client/lib/lib_test.go | 70 ++++---- client/table/table.go | 49 +----- client/testutils.go | 63 ++------ client/tui/tui.go | 71 -------- shared/testutils/testutils.go | 39 +++-- 17 files changed, 254 insertions(+), 567 deletions(-) diff --git a/backend/server/internal/database/db.go b/backend/server/internal/database/db.go index d151f36..b9804bf 100644 --- a/backend/server/internal/database/db.go +++ b/backend/server/internal/database/db.go @@ -4,8 +4,6 @@ import ( "context" "database/sql" "fmt" - "strings" - "time" "github.com/ddworken/hishtory/shared" "github.com/jackc/pgx/v4/stdlib" @@ -52,13 +50,14 @@ func (db *DB) AddDatabaseTables() error { &shared.DumpRequest{}, &shared.DeletionRequest{}, &shared.Feedback{}, - &ActiveUserStats{}, } for _, model := range models { + fmt.Printf("Beginning migration of %#v\n", model) if err := db.AutoMigrate(model); err != nil { return fmt.Errorf("db.AutoMigrate: %w", err) } + fmt.Printf("Done migrating %#v\n", model) } return nil @@ -68,25 +67,15 @@ func (db *DB) CreateIndices() error { // Note: If adding a new index here, consider manually running it on the prod DB using CONCURRENTLY to // make server startup non-blocking. The benefit of this function is primarily for other people so they // don't have to manually create these indexes. - indices := []struct { - name string - table string - columns []string - }{ - {"entry_id_idx", "enc_history_entries", []string{"encrypted_id"}}, - {"device_id_idx", "enc_history_entries", []string{"device_id"}}, - {"read_count_idx", "enc_history_entries", []string{"read_count"}}, - {"redact_idx", "enc_history_entries", []string{"user_id", "device_id", "date"}}, - {"del_user_idx", "deletion_requests", []string{"user_id"}}, + var indices = []string{ + `CREATE INDEX IF NOT EXISTS entry_id_idx ON enc_history_entries USING btree(encrypted_id);`, + `CREATE INDEX IF NOT EXISTS device_id_idx ON enc_history_entries USING btree(device_id);`, + `CREATE INDEX IF NOT EXISTS read_count_idx ON enc_history_entries USING btree(read_count);`, + `CREATE INDEX IF NOT EXISTS redact_idx ON enc_history_entries USING btree(user_id, device_id, date);`, + `CREATE INDEX IF NOT EXISTS del_user_idx ON deletion_requests USING btree(user_id);`, } for _, index := range indices { - sql := "" - if db.Name() == "sqlite" { - sql = fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (%s)", index.name, index.table, strings.Join(index.columns, ",")) - } else { - sql = fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s USING btree(%s)", index.name, index.table, strings.Join(index.columns, ",")) - } - r := db.Exec(sql) + r := db.Exec(index) if r.Error != nil { return fmt.Errorf("failed to execute index creation sql=%#v: %w", index, r.Error) } @@ -266,79 +255,6 @@ func (db *DB) Clean(ctx context.Context) error { return nil } -func extractInt64FromRow(row *sql.Row) (int64, error) { - var ret int64 - err := row.Scan(&ret) - if err != nil { - return 0, fmt.Errorf("extractInt64FromRow: %w", err) - } - return ret, nil -} - -type ActiveUserStats struct { - Date time.Time - TotalNumDevices int64 - TotalNumUsers int64 - DailyActiveSubmitUsers int64 - DailyActiveQueryUsers int64 - WeeklyActiveSubmitUsers int64 - WeeklyActiveQueryUsers int64 - DailyInstalls int64 - DailyUninstalls int64 -} - -func (db *DB) GenerateAndStoreActiveUserStats(ctx context.Context) error { - if db.DB.Name() == "sqlite" { - // Not supported on sqlite - return nil - } - - totalNumDevices, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT devices.device_id) FROM devices").Row()) - if err != nil { - return err - } - totalNumUsers, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT devices.user_id) FROM devices").Row()) - if err != nil { - return err - } - dauSubmit, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_used > (now()::date-1)::timestamp").Row()) - if err != nil { - return err - } - dauQuery, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_queried > (now()::date-1)::timestamp").Row()) - if err != nil { - return err - } - wauSubmit, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_used > (now()::date-7)::timestamp").Row()) - if err != nil { - return err - } - wauQuery, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(DISTINCT user_id) FROM usage_data WHERE last_queried > (now()::date-7)::timestamp").Row()) - if err != nil { - return err - } - dailyInstalls, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT count(distinct device_id) FROM devices WHERE registration_date > (now()::date-1)::timestamp").Row()) - if err != nil { - return err - } - dailyUninstalls, err := extractInt64FromRow(db.WithContext(ctx).Raw("SELECT COUNT(*) FROM feedbacks WHERE date > (now()::date-1)::timestamp").Row()) - if err != nil { - return err - } - - return db.Create(ActiveUserStats{ - Date: time.Now(), - TotalNumDevices: totalNumDevices, - TotalNumUsers: totalNumUsers, - DailyActiveSubmitUsers: dauSubmit, - DailyActiveQueryUsers: dauQuery, - WeeklyActiveSubmitUsers: wauSubmit, - WeeklyActiveQueryUsers: wauQuery, - DailyInstalls: dailyInstalls, - DailyUninstalls: dailyUninstalls, - }).Error -} - func (db *DB) DeepClean(ctx context.Context) error { return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { r := tx.Exec(` diff --git a/backend/server/internal/server/server_test.go b/backend/server/internal/server/server_test.go index 9e22a86..7009474 100644 --- a/backend/server/internal/server/server_test.go +++ b/backend/server/internal/server/server_test.go @@ -74,9 +74,9 @@ func TestESubmitThenQuery(t *testing.T) { // Submit a few entries for different devices entry := testutils.MakeFakeHistoryEntry("ls ~/") encEntry, err := data.EncryptHistoryEntry("key", entry) - require.NoError(t, err) + testutils.Check(t, err) reqBody, err := json.Marshal([]shared.EncHistoryEntry{encEntry}) - require.NoError(t, err) + testutils.Check(t, err) submitReq := httptest.NewRequest(http.MethodPost, "/?source_device_id="+devId1, bytes.NewReader(reqBody)) w := httptest.NewRecorder() s.apiSubmitHandler(w, submitReq) @@ -92,16 +92,16 @@ func TestESubmitThenQuery(t *testing.T) { res := w.Result() defer res.Body.Close() respBody, err := io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) var retrievedEntries []*shared.EncHistoryEntry - require.NoError(t, json.Unmarshal(respBody, &retrievedEntries)) + testutils.Check(t, json.Unmarshal(respBody, &retrievedEntries)) require.Equal(t, 1, len(retrievedEntries)) dbEntry := retrievedEntries[0] require.Equal(t, devId1, dbEntry.DeviceId) require.Equal(t, data.UserId("key"), dbEntry.UserId) require.Equal(t, 0, dbEntry.ReadCount) decEntry, err := data.DecryptHistoryEntry("key", *dbEntry) - require.NoError(t, err) + testutils.Check(t, err) require.True(t, data.EntryEquals(decEntry, entry)) // Same for device id 2 @@ -111,8 +111,8 @@ func TestESubmitThenQuery(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) - require.NoError(t, json.Unmarshal(respBody, &retrievedEntries)) + testutils.Check(t, err) + testutils.Check(t, json.Unmarshal(respBody, &retrievedEntries)) if len(retrievedEntries) != 1 { t.Fatalf("Expected to retrieve 1 entry, found %d", len(retrievedEntries)) } @@ -127,7 +127,7 @@ func TestESubmitThenQuery(t *testing.T) { t.Fatalf("db.ReadCount should have been 1, was %v", dbEntry.ReadCount) } decEntry, err = data.DecryptHistoryEntry("key", *dbEntry) - require.NoError(t, err) + testutils.Check(t, err) if !data.EntryEquals(decEntry, entry) { t.Fatalf("DB data is different than input! \ndb =%#v\ninput=%#v", *dbEntry, entry) } @@ -139,8 +139,8 @@ func TestESubmitThenQuery(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) - require.NoError(t, json.Unmarshal(respBody, &retrievedEntries)) + testutils.Check(t, err) + testutils.Check(t, json.Unmarshal(respBody, &retrievedEntries)) if len(retrievedEntries) != 2 { t.Fatalf("Expected to retrieve 2 entries, found %d", len(retrievedEntries)) } @@ -175,9 +175,9 @@ func TestDumpRequestAndResponse(t *testing.T) { res := w.Result() defer res.Body.Close() respBody, err := io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) var dumpRequests []*shared.DumpRequest - require.NoError(t, json.Unmarshal(respBody, &dumpRequests)) + testutils.Check(t, json.Unmarshal(respBody, &dumpRequests)) if len(dumpRequests) != 1 { t.Fatalf("expected one pending dump request, got %#v", dumpRequests) } @@ -195,9 +195,9 @@ func TestDumpRequestAndResponse(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) dumpRequests = make([]*shared.DumpRequest, 0) - require.NoError(t, json.Unmarshal(respBody, &dumpRequests)) + testutils.Check(t, json.Unmarshal(respBody, &dumpRequests)) if len(dumpRequests) != 1 { t.Fatalf("expected one pending dump request, got %#v", dumpRequests) } @@ -215,7 +215,7 @@ func TestDumpRequestAndResponse(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) resp := strings.TrimSpace(string(respBody)) require.Equalf(t, "[]", resp, "got unexpected respBody: %#v", string(resp)) @@ -225,19 +225,19 @@ func TestDumpRequestAndResponse(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) resp = strings.TrimSpace(string(respBody)) require.Equalf(t, "[]", resp, "got unexpected respBody: %#v", string(resp)) // Now submit a dump for userId entry1Dec := testutils.MakeFakeHistoryEntry("ls ~/") entry1, err := data.EncryptHistoryEntry("dkey", entry1Dec) - require.NoError(t, err) + testutils.Check(t, err) entry2Dec := testutils.MakeFakeHistoryEntry("aaaaaaáaaa") entry2, err := data.EncryptHistoryEntry("dkey", entry1Dec) - require.NoError(t, err) + testutils.Check(t, err) reqBody, err := json.Marshal([]shared.EncHistoryEntry{entry1, entry2}) - require.NoError(t, err) + testutils.Check(t, err) submitReq := httptest.NewRequest(http.MethodPost, "/?user_id="+userId+"&requesting_device_id="+devId2+"&source_device_id="+devId1, bytes.NewReader(reqBody)) s.apiSubmitDumpHandler(httptest.NewRecorder(), submitReq) @@ -247,7 +247,7 @@ func TestDumpRequestAndResponse(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) resp = strings.TrimSpace(string(respBody)) require.Equalf(t, "[]", resp, "got unexpected respBody: %#v", string(respBody)) @@ -258,7 +258,7 @@ func TestDumpRequestAndResponse(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) resp = strings.TrimSpace(string(respBody)) require.Equalf(t, "[]", resp, "got unexpected respBody: %#v", string(respBody)) @@ -268,9 +268,9 @@ func TestDumpRequestAndResponse(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) dumpRequests = make([]*shared.DumpRequest, 0) - require.NoError(t, json.Unmarshal(respBody, &dumpRequests)) + testutils.Check(t, json.Unmarshal(respBody, &dumpRequests)) if len(dumpRequests) != 1 { t.Fatalf("expected one pending dump request, got %#v", dumpRequests) } @@ -289,9 +289,9 @@ func TestDumpRequestAndResponse(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) var retrievedEntries []*shared.EncHistoryEntry - require.NoError(t, json.Unmarshal(respBody, &retrievedEntries)) + testutils.Check(t, json.Unmarshal(respBody, &retrievedEntries)) if len(retrievedEntries) != 2 { t.Fatalf("Expected to retrieve 2 entries, found %d", len(retrievedEntries)) } @@ -306,7 +306,7 @@ func TestDumpRequestAndResponse(t *testing.T) { t.Fatalf("db.ReadCount should have been 1, was %v", dbEntry.ReadCount) } decEntry, err := data.DecryptHistoryEntry("dkey", *dbEntry) - require.NoError(t, err) + testutils.Check(t, err) if !data.EntryEquals(decEntry, entry1Dec) && !data.EntryEquals(decEntry, entry2Dec) { t.Fatalf("DB data is different than input! \ndb =%#v\nentry1=%#v\nentry2=%#v", *dbEntry, entry1Dec, entry2Dec) } @@ -340,9 +340,9 @@ func TestDeletionRequests(t *testing.T) { entry1 := testutils.MakeFakeHistoryEntry("ls ~/") entry1.DeviceId = devId1 encEntry, err := data.EncryptHistoryEntry("dkey", entry1) - require.NoError(t, err) + testutils.Check(t, err) reqBody, err := json.Marshal([]shared.EncHistoryEntry{encEntry}) - require.NoError(t, err) + testutils.Check(t, err) submitReq := httptest.NewRequest(http.MethodPost, "/?source_device_id="+devId1, bytes.NewReader(reqBody)) w := httptest.NewRecorder() s.apiSubmitHandler(w, submitReq) @@ -354,9 +354,9 @@ func TestDeletionRequests(t *testing.T) { entry2 := testutils.MakeFakeHistoryEntry("ls /foo/bar") entry2.DeviceId = devId2 encEntry, err = data.EncryptHistoryEntry("dkey", entry2) - require.NoError(t, err) + testutils.Check(t, err) reqBody, err = json.Marshal([]shared.EncHistoryEntry{encEntry}) - require.NoError(t, err) + testutils.Check(t, err) submitReq = httptest.NewRequest(http.MethodPost, "/?source_device_id="+devId2, bytes.NewReader(reqBody)) w = httptest.NewRecorder() s.apiSubmitHandler(w, submitReq) @@ -369,9 +369,9 @@ func TestDeletionRequests(t *testing.T) { entry3.StartTime = entry1.StartTime entry3.EndTime = entry1.EndTime encEntry, err = data.EncryptHistoryEntry("dOtherkey", entry3) - require.NoError(t, err) + testutils.Check(t, err) reqBody, err = json.Marshal([]shared.EncHistoryEntry{encEntry}) - require.NoError(t, err) + testutils.Check(t, err) submitReq = httptest.NewRequest(http.MethodPost, "/?source_device_id="+devId1, bytes.NewReader(reqBody)) w = httptest.NewRecorder() s.apiSubmitHandler(w, submitReq) @@ -386,9 +386,9 @@ func TestDeletionRequests(t *testing.T) { res := w.Result() defer res.Body.Close() respBody, err := io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) var retrievedEntries []*shared.EncHistoryEntry - require.NoError(t, json.Unmarshal(respBody, &retrievedEntries)) + testutils.Check(t, json.Unmarshal(respBody, &retrievedEntries)) if len(retrievedEntries) != 2 { t.Fatalf("Expected to retrieve 1 entry, found %d", len(retrievedEntries)) } @@ -403,7 +403,7 @@ func TestDeletionRequests(t *testing.T) { t.Fatalf("db.ReadCount should have been 1, was %v", dbEntry.ReadCount) } decEntry, err := data.DecryptHistoryEntry("dkey", *dbEntry) - require.NoError(t, err) + testutils.Check(t, err) if !data.EntryEquals(decEntry, entry1) && !data.EntryEquals(decEntry, entry2) { t.Fatalf("DB data is different than input! \ndb =%#v\nentry1=%#v\nentry2=%#v", *dbEntry, entry1, entry2) } @@ -419,7 +419,7 @@ func TestDeletionRequests(t *testing.T) { }}, } reqBody, err = json.Marshal(delReq) - require.NoError(t, err) + testutils.Check(t, err) req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(reqBody)) s.addDeletionRequestHandler(httptest.NewRecorder(), req) @@ -431,8 +431,8 @@ func TestDeletionRequests(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) - require.NoError(t, json.Unmarshal(respBody, &retrievedEntries)) + testutils.Check(t, err) + testutils.Check(t, json.Unmarshal(respBody, &retrievedEntries)) if len(retrievedEntries) != 1 { t.Fatalf("Expected to retrieve 1 entry, found %d", len(retrievedEntries)) } @@ -447,7 +447,7 @@ func TestDeletionRequests(t *testing.T) { t.Fatalf("db.ReadCount should have been 1, was %v", dbEntry.ReadCount) } decEntry, err := data.DecryptHistoryEntry("dkey", *dbEntry) - require.NoError(t, err) + testutils.Check(t, err) if !data.EntryEquals(decEntry, entry2) { t.Fatalf("DB data is different than input! \ndb =%#v\nentry=%#v", *dbEntry, entry2) } @@ -459,8 +459,8 @@ func TestDeletionRequests(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) - require.NoError(t, json.Unmarshal(respBody, &retrievedEntries)) + testutils.Check(t, err) + testutils.Check(t, json.Unmarshal(respBody, &retrievedEntries)) if len(retrievedEntries) != 1 { t.Fatalf("Expected to retrieve 1 entry, found %d", len(retrievedEntries)) } @@ -475,16 +475,16 @@ func TestDeletionRequests(t *testing.T) { t.Fatalf("db.ReadCount should have been 1, was %v", dbEntry.ReadCount) } decEntry, err = data.DecryptHistoryEntry("dOtherkey", *dbEntry) - require.NoError(t, err) + testutils.Check(t, err) if !data.EntryEquals(decEntry, entry3) { t.Fatalf("DB data is different than input! \ndb =%#v\nentry=%#v", *dbEntry, entry3) } // Check that apiSubmit tells the client that there is a pending deletion request encEntry, err = data.EncryptHistoryEntry("dkey", entry2) - require.NoError(t, err) + testutils.Check(t, err) reqBody, err = json.Marshal([]shared.EncHistoryEntry{encEntry}) - require.NoError(t, err) + testutils.Check(t, err) submitReq = httptest.NewRequest(http.MethodPost, "/?source_device_id="+devId2, bytes.NewReader(reqBody)) w = httptest.NewRecorder() s.apiSubmitHandler(w, submitReq) @@ -499,9 +499,9 @@ func TestDeletionRequests(t *testing.T) { res = w.Result() defer res.Body.Close() respBody, err = io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) var deletionRequests []*shared.DeletionRequest - require.NoError(t, json.Unmarshal(respBody, &deletionRequests)) + testutils.Check(t, json.Unmarshal(respBody, &deletionRequests)) if len(deletionRequests) != 1 { t.Fatalf("received %d deletion requests, expected only one", len(deletionRequests)) } @@ -533,7 +533,7 @@ func TestHealthcheck(t *testing.T) { res := w.Result() defer res.Body.Close() respBody, err := io.ReadAll(res.Body) - require.NoError(t, err) + testutils.Check(t, err) if string(respBody) != "OK" { t.Fatalf("expected healthcheckHandler to return OK") } @@ -583,9 +583,9 @@ func TestCleanDatabaseNoErrors(t *testing.T) { entry1 := testutils.MakeFakeHistoryEntry("ls ~/") entry1.DeviceId = devId1 encEntry, err := data.EncryptHistoryEntry("dkey", entry1) - require.NoError(t, err) + testutils.Check(t, err) reqBody, err := json.Marshal([]shared.EncHistoryEntry{encEntry}) - require.NoError(t, err) + testutils.Check(t, err) submitReq := httptest.NewRequest(http.MethodPost, "/?source_device_id="+devId1, bytes.NewReader(reqBody)) w := httptest.NewRecorder() s.apiSubmitHandler(w, submitReq) @@ -594,7 +594,7 @@ func TestCleanDatabaseNoErrors(t *testing.T) { require.NotEmpty(t, deserializeSubmitResponse(t, w).DumpRequests) // Call cleanDatabase and just check that there are no panics - require.NoError(t, DB.Clean(context.TODO())) + testutils.Check(t, DB.Clean(context.TODO())) } func assertNoLeakedConnections(t *testing.T, db *database.DB) { diff --git a/backend/server/internal/server/srv.go b/backend/server/internal/server/srv.go index d45b4ee..e40e442 100644 --- a/backend/server/internal/server/srv.go +++ b/backend/server/internal/server/srv.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net/http" - "strings" "time" "github.com/DataDog/datadog-go/statsd" @@ -153,7 +152,7 @@ func (s *Server) updateUsageData(ctx context.Context, version string, remoteAddr } var usageData []shared.UsageData usageData, err := s.db.UsageDataFindByUserAndDevice(ctx, userId, deviceId) - if err != nil && !strings.Contains(err.Error(), "record not found") { + if err != nil { return fmt.Errorf("db.UsageDataFindByUserAndDevice: %w", err) } if len(usageData) == 0 { diff --git a/backend/server/server.go b/backend/server/server.go index f9e2708..bae2833 100644 --- a/backend/server/server.go +++ b/backend/server/server.go @@ -102,36 +102,19 @@ func OpenDB() (*database.DB, error) { return db, nil } -var CRON_COUNTER = 0 - func cron(ctx context.Context, db *database.DB, stats *statsd.Client) error { - // Determine the latest released version of hishtory to serve via the /api/v1/download - // endpoint for hishtory updates. if err := release.UpdateReleaseVersion(); err != nil { return fmt.Errorf("updateReleaseVersion: %w", err) } - // Clean the DB to remove entries that have already been read if err := db.Clean(ctx); err != nil { return fmt.Errorf("db.Clean: %w", err) } - - // Flush out datadog statsd if stats != nil { if err := stats.Flush(); err != nil { return fmt.Errorf("stats.Flush: %w", err) } } - - // Collect and store metrics on active users so we can track trends over time. This doesn't - // have to be run as often, so only run it for a fraction of cron jobs - if CRON_COUNTER%40 == 0 { - if err := db.GenerateAndStoreActiveUserStats(ctx); err != nil { - return fmt.Errorf("db.GenerateAndStoreActiveUserStats: %w", err) - } - } - - CRON_COUNTER += 1 return nil } @@ -140,7 +123,10 @@ func runBackgroundJobs(ctx context.Context, srv *server.Server, db *database.DB, for { err := cron(ctx, db, stats) if err != nil { - panic(fmt.Sprintf("Cron failure: %v", err)) + fmt.Printf("Cron failure: %v", err) + + // cron no longer panics, panicking here. + panic(err) } srv.UpdateReleaseVersion(release.Version, release.BuildUpdateInfo(release.Version)) time.Sleep(10 * time.Minute) diff --git a/client/client_test.go b/client/client_test.go index 4b631f7..340f891 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -111,7 +111,6 @@ func TestParam(t *testing.T) { runTestsWithRetries(t, "testTui/scroll", testTui_scroll) runTestsWithRetries(t, "testTui/resize", testTui_resize) runTestsWithRetries(t, "testTui/delete", testTui_delete) - runTestsWithRetries(t, "testTui/color", testTui_color) // Assert there are no leaked connections assertNoLeakedConnections(t) @@ -169,7 +168,7 @@ func testIntegrationWithNewDevice(t *testing.T, tester shellTester) { // Set the secret key to the previous secret key out, err := tester.RunInteractiveShellRelaxed(t, ` export HISHTORY_SKIP_INIT_IMPORT=1 yes | hishtory init `+userSecret) - require.NoError(t, err) + testutils.Check(t, err) require.Contains(t, out, "Setting secret hishtory key to "+userSecret, "Failed to re-init with the user secret") // Querying shouldn't show the entry from the previous account @@ -298,22 +297,22 @@ echo thisisrecorded`) line2Matcher := hostnameMatcher + tableDividerMatcher + pathMatcher + tableDividerMatcher + datetimeMatcher + tableDividerMatcher + runtimeMatcher + tableDividerMatcher + exitCodeMatcher + tableDividerMatcher + pipefailMatcher + tableDividerMatcher + `\n` line3Matcher := hostnameMatcher + tableDividerMatcher + pathMatcher + tableDividerMatcher + datetimeMatcher + tableDividerMatcher + runtimeMatcher + tableDividerMatcher + exitCodeMatcher + tableDividerMatcher + `echo thisisrecorded` + tableDividerMatcher + `\n` match, err := regexp.MatchString(line3Matcher, out) - require.NoError(t, err) + testutils.Check(t, err) if !match { t.Fatalf("output is missing the row for `echo thisisrecorded`: %v", out) } match, err = regexp.MatchString(line1Matcher, out) - require.NoError(t, err) + testutils.Check(t, err) if !match { t.Fatalf("output is missing the headings: %v", out) } match, err = regexp.MatchString(line2Matcher, out) - require.NoError(t, err) + testutils.Check(t, err) if !match { t.Fatalf("output is missing the pipefail: %v", out) } match, err = regexp.MatchString(line1Matcher+line2Matcher+line3Matcher, out) - require.NoError(t, err) + testutils.Check(t, err) if !match { t.Fatalf("output doesn't match the expected table: %v", out) } @@ -791,7 +790,7 @@ func testHishtoryBackgroundSaving(t *testing.T, tester shellTester) { // Check that we can find the go binary _, err := exec.LookPath("go") - require.NoError(t, err) + testutils.Check(t, err) // Test install with an unset HISHTORY_TEST var so that we save in the background (this is likely to be flakey!) out := tester.RunInteractiveShell(t, `unset HISHTORY_TEST @@ -890,7 +889,7 @@ func testDisplayTable(t *testing.T, tester shellTester) { // Add a custom column tester.RunInteractiveShell(t, `hishtory config-add custom-columns foo "echo aaaaaaaaaaaaa"`) - require.NoError(t, os.Chdir("/")) + testutils.Check(t, os.Chdir("/")) tester.RunInteractiveShell(t, ` hishtory enable`) tester.RunInteractiveShell(t, `echo table-1`) tester.RunInteractiveShell(t, `echo table-2`) @@ -1356,7 +1355,7 @@ ls /tmp`, randomCmdUuid, randomCmdUuid) // Redact it without HISHTORY_REDACT_FORCE out, err := tester.RunInteractiveShellRelaxed(t, `yes | hishtory redact hello`) - require.NoError(t, err) + testutils.Check(t, err) if out != "This will permanently delete 1 entries, are you sure? [y/N]" { t.Fatalf("hishtory redact gave unexpected output=%#v", out) } @@ -1474,12 +1473,12 @@ func testConfigGetSet(t *testing.T, tester shellTester) { func clearControlRSearchFromConfig(t testing.TB) { configContents, err := hctx.GetConfigContents() - require.NoError(t, err) + testutils.Check(t, err) configContents = []byte(strings.ReplaceAll(string(configContents), "enable_control_r_search", "something-else")) homedir, err := os.UserHomeDir() - require.NoError(t, err) + testutils.Check(t, err) err = os.WriteFile(path.Join(homedir, data.GetHishtoryPath(), data.CONFIG_PATH), configContents, 0o644) - require.NoError(t, err) + testutils.Check(t, err) } func testHandleUpgradedFeatures(t *testing.T, tester shellTester) { @@ -1489,9 +1488,9 @@ func testHandleUpgradedFeatures(t *testing.T, tester shellTester) { // Install, and there is no prompt since the config already mentions control-r _, err := tester.RunInteractiveShellRelaxed(t, `/tmp/client install`) - require.NoError(t, err) + testutils.Check(t, err) _, err = tester.RunInteractiveShellRelaxed(t, `hishtory disable`) - require.NoError(t, err) + testutils.Check(t, err) // Ensure that the config doesn't mention control-r clearControlRSearchFromConfig(t) @@ -1520,7 +1519,7 @@ func TestFish(t *testing.T) { installHishtory(t, tester, "") // Test recording in fish - require.NoError(t, os.Chdir("/")) + testutils.Check(t, os.Chdir("/")) out := captureTerminalOutputWithShellName(t, tester, "fish", []string{ "echo SPACE foo ENTER", "ENTER", @@ -1559,10 +1558,10 @@ func setupTestTui(t testing.TB) (shellTester, string, *gorm.DB) { // Insert a couple hishtory entries db := hctx.GetDb(hctx.MakeContext()) e1 := testutils.MakeFakeHistoryEntry("ls ~/") - require.NoError(t, db.Create(e1).Error) + testutils.Check(t, db.Create(e1).Error) manuallySubmitHistoryEntry(t, userSecret, e1) e2 := testutils.MakeFakeHistoryEntry("echo 'aaaaaa bbbb'") - require.NoError(t, db.Create(e2).Error) + testutils.Check(t, db.Create(e2).Error) manuallySubmitHistoryEntry(t, userSecret, e2) return tester, userSecret, db } @@ -1605,6 +1604,9 @@ func testTui_resize(t testing.TB) { }) out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) testutils.CompareGoldens(t, out, "TestTui-LongQuery") + + // Assert there are no leaked connections + assertNoLeakedConnections(t) } func testTui_scroll(t testing.TB) { @@ -1644,36 +1646,6 @@ func testTui_scroll(t testing.TB) { } -func testTui_color(t testing.TB) { - if runtime.GOOS == "linux" { - // For some reason, this test fails on linux. Since this test isn't critical and is expected to be - // flaky, we can just skip it on linux. - t.Skip() - } - - // Setup - defer testutils.BackupAndRestore(t)() - tester, _, _ := setupTestTui(t) - - // Capture the TUI with full colored output, note that this golden will be harder to undersand - // from inspection and primarily servers to detect unintended changes in hishtory's output. - out := captureTerminalOutputComplex(t, TmuxCaptureConfig{tester: tester, complexCommands: []TmuxCommand{{Keys: "hishtory SPACE tquery ENTER"}}, includeEscapeSequences: true}) - out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) - testutils.CompareGoldens(t, out, "TestTui-ColoredOutput") - - // And the same once a search query has been typed in - out = captureTerminalOutputComplex(t, TmuxCaptureConfig{tester: tester, complexCommands: []TmuxCommand{{Keys: "hishtory SPACE tquery ENTER"}, {Keys: "ech"}}, includeEscapeSequences: true}) - out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) - testutils.CompareGoldens(t, out, "TestTui-ColoredOutputWithSearch") - - // And one more time with beta-mode for highlighting matches - tester.RunInteractiveShell(t, ` hishtory config-set beta-mode true`) - require.Equal(t, "true", strings.TrimSpace(tester.RunInteractiveShell(t, `hishtory config-get beta-mode`))) - out = captureTerminalOutputComplex(t, TmuxCaptureConfig{tester: tester, complexCommands: []TmuxCommand{{Keys: "hishtory SPACE tquery ENTER"}, {Keys: "ech"}}, includeEscapeSequences: true}) - out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) - testutils.CompareGoldens(t, out, "TestTui-ColoredOutputWithSearch-BetaMode") -} - func testTui_delete(t testing.TB) { // Setup defer testutils.BackupAndRestore(t)() @@ -1752,7 +1724,7 @@ func testTui_search(t testing.TB) { // Check the output when the initial search is invalid out = captureTerminalOutputWithComplexCommands(t, tester, []TmuxCommand{ // ExtraDelay to ensure that after searching for 'foo:' it gets the real results for an empty query - {Keys: "hishtory SPACE tquery SPACE foo: ENTER", ExtraDelay: 1.5}, + {Keys: "hishtory SPACE tquery SPACE foo: ENTER", ExtraDelay: 1.0}, {Keys: "ls", ExtraDelay: 1.0}, }) out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) @@ -1762,7 +1734,7 @@ func testTui_search(t testing.TB) { out = captureTerminalOutputWithComplexCommands(t, tester, []TmuxCommand{ {Keys: "hishtory SPACE tquery ENTER"}, // ExtraDelay to ensure that the search for 'ls' finishes before we make it invalid by adding ':' - {Keys: "ls", ExtraDelay: 1.5}, + {Keys: "ls", ExtraDelay: 1.0}, {Keys: ":"}, }) out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) @@ -1775,6 +1747,7 @@ func testTui_search(t testing.TB) { }) out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) testutils.CompareGoldens(t, out, "TestTui-InvalidSearchBecomesValid") + } func testTui_general(t testing.TB) { @@ -1853,11 +1826,11 @@ func testControlR(t testing.TB, tester shellTester, shellName string, onlineStat e1.CurrentWorkingDirectory = "/etc/" e1.Hostname = "server" e1.ExitCode = 127 - require.NoError(t, db.Create(e1).Error) - require.NoError(t, db.Create(testutils.MakeFakeHistoryEntry("ls ~/foo/")).Error) - require.NoError(t, db.Create(testutils.MakeFakeHistoryEntry("ls ~/bar/")).Error) - require.NoError(t, db.Create(testutils.MakeFakeHistoryEntry("echo 'aaaaaa bbbb'")).Error) - require.NoError(t, db.Create(testutils.MakeFakeHistoryEntry("echo 'bar' &")).Error) + testutils.Check(t, db.Create(e1).Error) + testutils.Check(t, db.Create(testutils.MakeFakeHistoryEntry("ls ~/foo/")).Error) + testutils.Check(t, db.Create(testutils.MakeFakeHistoryEntry("ls ~/bar/")).Error) + testutils.Check(t, db.Create(testutils.MakeFakeHistoryEntry("echo 'aaaaaa bbbb'")).Error) + testutils.Check(t, db.Create(testutils.MakeFakeHistoryEntry("echo 'bar' &")).Error) // Check that they're there var historyEntries []*data.HistoryEntry @@ -1992,7 +1965,7 @@ func testControlR(t testing.TB, tester shellTester, shellName string, onlineStat // Re-enable control-r _, err := tester.RunInteractiveShellRelaxed(t, `hishtory config-set enable-control-r true`) - require.NoError(t, err) + testutils.Check(t, err) // And check that the control-r bindings work again out = captureTerminalOutputWithShellName(t, tester, shellName, []string{"C-R", "-pipefail SPACE -exit_code:0"}) @@ -2072,20 +2045,17 @@ echo bar`) } func testPresaving(t *testing.T, tester shellTester) { - if testutils.IsGithubAction() && tester.ShellName() == "bash" { - // TODO: Debug the issues with presaving and bash, and re-enable this test - t.Skip() - } - // Setup defer testutils.BackupAndRestore(t)() userSecret := installHishtory(t, tester, "") manuallySubmitHistoryEntry(t, userSecret, testutils.MakeFakeHistoryEntry("table_sizing aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) // Enable beta-mode since presaving is behind that feature flag - require.Equal(t, "false", strings.TrimSpace(tester.RunInteractiveShell(t, `hishtory config-get beta-mode`))) + out := strings.TrimSpace(tester.RunInteractiveShell(t, `hishtory config-get beta-mode`)) + require.Equal(t, out, "false") tester.RunInteractiveShell(t, `hishtory config-set beta-mode true`) - require.Equal(t, "true", strings.TrimSpace(tester.RunInteractiveShell(t, `hishtory config-get beta-mode`))) + out = strings.TrimSpace(tester.RunInteractiveShell(t, `hishtory config-get beta-mode`)) + require.Equal(t, out, "true") // Start a command that will take a long time to execute in the background, so // we can check that it was recorded even though it never finished. @@ -2094,7 +2064,7 @@ func testPresaving(t *testing.T, tester shellTester) { time.Sleep(time.Millisecond * 500) // Test that it shows up in hishtory export - out := tester.RunInteractiveShell(t, ` hishtory export sleep -export`) + out = tester.RunInteractiveShell(t, ` hishtory export sleep -export`) expectedOutput := "sleep 13371337\n" if diff := cmp.Diff(expectedOutput, out); diff != "" { t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out) @@ -2105,16 +2075,6 @@ func testPresaving(t *testing.T, tester shellTester) { out = tester.RunInteractiveShell(t, ` hishtory query sleep 13371337 -export -tquery`) testutils.CompareGoldens(t, out, "testPresaving-query") - // And then record a few other commands, and run an export of all commands, to ensure no funkiness happened - tester.RunInteractiveShell(t, `ls /`) - time.Sleep(time.Second) - tester.RunInteractiveShell(t, `sleep 0.5`) - out = tester.RunInteractiveShell(t, ` hishtory export -hishtory -table_sizing -pipefail`) - expectedOutput = "sleep 13371337\nls /\nsleep 0.5\n" - if diff := cmp.Diff(expectedOutput, out); diff != "" { - t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out) - } - // Create a new device, and confirm it shows up there too restoreDevice1 := testutils.BackupAndRestoreWithId(t, "device1") installHishtory(t, tester, userSecret) @@ -2122,31 +2082,17 @@ func testPresaving(t *testing.T, tester shellTester) { out = tester.RunInteractiveShell(t, ` hishtory query sleep 13371337 -export -tquery`) testutils.CompareGoldens(t, out, "testPresaving-query") - // And that all the other commands do to - out = tester.RunInteractiveShell(t, ` hishtory export -hishtory -table_sizing -pipefail`) - expectedOutput = "sleep 13371337\nls /\nsleep 0.5\n" - if diff := cmp.Diff(expectedOutput, out); diff != "" { - t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out) - } - // And then redact it from device2 tester.RunInteractiveShell(t, ` HISHTORY_REDACT_FORCE=true hishtory redact sleep 13371337`) // And confirm it was redacted out = tester.RunInteractiveShell(t, ` hishtory export sleep -export`) - require.Equal(t, "sleep 0.5\n", out) + require.Equal(t, "", out) // Then go back to device1 and confirm it was redacted there too restoreDevice1() out = tester.RunInteractiveShell(t, ` hishtory export sleep -export`) - require.Equal(t, "sleep 0.5\n", out) - - // And then record a few commands, and run a final export of all commands, to ensure no funkiness happened - out = tester.RunInteractiveShell(t, ` hishtory export -hishtory -table_sizing -pipefail`) - expectedOutput = "ls /\nsleep 0.5\n" - if diff := cmp.Diff(expectedOutput, out); diff != "" { - t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out) - } + require.Equal(t, "", out) } func testUninstall(t *testing.T, tester shellTester) { @@ -2162,14 +2108,14 @@ echo baz`) // And then uninstall out, err := tester.RunInteractiveShellRelaxed(t, `yes | hishtory uninstall`) - require.NoError(t, err) + testutils.Check(t, err) testutils.CompareGoldens(t, out, "testUninstall-uninstall") // And check that hishtory has been uninstalled out, err = tester.RunInteractiveShellRelaxed(t, `echo foo hishtory echo bar`) - require.NoError(t, err) + testutils.Check(t, err) testutils.CompareGoldens(t, out, "testUninstall-post-uninstall") // And check again, but in a way that shows the full terminal output @@ -2236,15 +2182,15 @@ func TestSortByConsistentTimezone(t *testing.T) { entry1 := testutils.MakeFakeHistoryEntry("first_entry") entry1.StartTime = time.Unix(timestamp, 0).In(ny_time) entry1.EndTime = time.Unix(timestamp+1, 0).In(ny_time) - require.NoError(t, lib.ReliableDbCreate(db, entry1)) + testutils.Check(t, lib.ReliableDbCreate(db, entry1)) entry2 := testutils.MakeFakeHistoryEntry("second_entry") entry2.StartTime = time.Unix(timestamp+1000, 0).In(la_time) entry2.EndTime = time.Unix(timestamp+1001, 0).In(la_time) - require.NoError(t, lib.ReliableDbCreate(db, entry2)) + testutils.Check(t, lib.ReliableDbCreate(db, entry2)) entry3 := testutils.MakeFakeHistoryEntry("third_entry") entry3.StartTime = time.Unix(timestamp+2000, 0).In(ny_time) entry3.EndTime = time.Unix(timestamp+2001, 0).In(ny_time) - require.NoError(t, lib.ReliableDbCreate(db, entry3)) + testutils.Check(t, lib.ReliableDbCreate(db, entry3)) // And check that they're displayed in the correct order out := hishtoryQuery(t, tester, "-pipefail -tablesizing") @@ -2262,13 +2208,13 @@ func TestZDotDir(t *testing.T) { defer testutils.BackupAndRestore(t)() defer testutils.BackupAndRestoreEnv("ZDOTDIR")() homedir, err := os.UserHomeDir() - require.NoError(t, err) + testutils.Check(t, err) zdotdir := path.Join(homedir, "foo") - require.NoError(t, os.MkdirAll(zdotdir, 0o744)) + testutils.Check(t, os.MkdirAll(zdotdir, 0o744)) os.Setenv("ZDOTDIR", zdotdir) userSecret := installHishtory(t, tester, "") defer func() { - require.NoError(t, os.Remove(path.Join(zdotdir, ".zshrc"))) + testutils.Check(t, os.Remove(path.Join(zdotdir, ".zshrc"))) }() // Check the status command @@ -2286,7 +2232,7 @@ func TestZDotDir(t *testing.T) { // Check that hishtory respected ZDOTDIR zshrc, err := os.ReadFile(path.Join(zdotdir, ".zshrc")) - require.NoError(t, err) + testutils.Check(t, err) require.Contains(t, string(zshrc), "# Hishtory Config:", "zshrc had unexpected contents") } @@ -2318,7 +2264,7 @@ echo foo`) out = tester.RunInteractiveShell(t, `hishtory query -pipefail`) testutils.CompareGoldens(t, out, "testRemoveDuplicateRows-enabled-query") out = captureTerminalOutput(t, tester, []string{"hishtory SPACE tquery SPACE -pipefail ENTER"}) - out = strings.TrimSpace(strings.Split(out, "hishtory tquery -pipefail")[1]) + out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) testutils.CompareGoldens(t, out, "testRemoveDuplicateRows-enabled-tquery") out = captureTerminalOutput(t, tester, []string{"hishtory SPACE tquery SPACE -pipefail ENTER", "Down Down", "ENTER"}) out = strings.TrimSpace(strings.Split(out, "hishtory tquery")[1]) @@ -2334,7 +2280,7 @@ func TestSetConfigNoCorruption(t *testing.T) { // A test that tries writing a config many different times in parallel, and confirms there is no corruption conf, err := hctx.GetConfig() - require.NoError(t, err) + testutils.Check(t, err) var doneWg sync.WaitGroup for i := 0; i < 10; i++ { doneWg.Add(1) @@ -2345,7 +2291,7 @@ func TestSetConfigNoCorruption(t *testing.T) { c.DeviceId = strings.Repeat("B", i*2) c.HaveMissedUploads = (i % 2) == 0 // Write it - require.NoError(t, hctx.SetConfig(&c)) + err := hctx.SetConfig(&c) require.NoError(t, err) // Check that we can read c2, err := hctx.GetConfig() @@ -2431,7 +2377,7 @@ func testMultipleUsers(t *testing.T, tester shellTester) { for _, d := range []device{u1d1, u1d2} { switchToDevice(&devices, d) out, err := tester.RunInteractiveShellRelaxed(t, `hishtory export -pipefail -export`) - require.NoError(t, err) + testutils.Check(t, err) expectedOutput := "echo u1d1\necho u1d2\necho u1d1-b\necho u1d1-c\necho u1d2-b\necho u1d2-c\n" if diff := cmp.Diff(expectedOutput, out); diff != "" { t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out) @@ -2450,7 +2396,7 @@ func testMultipleUsers(t *testing.T, tester shellTester) { for _, d := range []device{u2d1, u2d2, u2d3} { switchToDevice(&devices, d) out, err := tester.RunInteractiveShellRelaxed(t, `hishtory export -export -pipefail`) - require.NoError(t, err) + testutils.Check(t, err) expectedOutput := "echo u2d1\necho u2d2\necho u2d3\necho u1d1-b\necho u1d1-c\necho u2d3-b\necho u2d3-c\n" if diff := cmp.Diff(expectedOutput, out); diff != "" { t.Fatalf("hishtory export mismatch (-expected +got):\n%s\nout=%#v", diff, out) diff --git a/client/cmd/saveHistoryEntry.go b/client/cmd/saveHistoryEntry.go index 2563186..90dfe6c 100644 --- a/client/cmd/saveHistoryEntry.go +++ b/client/cmd/saveHistoryEntry.go @@ -144,11 +144,8 @@ func presaveHistoryEntry(ctx context.Context) { } // Augment it with os.Args - shellName := os.Args[2] - cmd, err := extractCommandFromArg(ctx, shellName, os.Args[3]) - lib.CheckFatalError(err) - entry.Command = cmd - if strings.HasPrefix(" ", entry.Command) || strings.TrimSpace(entry.Command) == "" { + entry.Command = trimTrailingWhitespace(os.Args[3]) + if strings.HasPrefix(" ", entry.Command) || entry.Command == "" { // Don't save commands that start with a space return } @@ -157,6 +154,11 @@ func presaveHistoryEntry(ctx context.Context) { entry.StartTime = time.Unix(startTime, 0).UTC() entry.EndTime = time.Unix(0, 0).UTC() + // Skip saving references to presaving + if strings.Contains(entry.Command, "presaveHistoryEntry") || strings.Contains(entry.Command, "__history_control_r") { + return + } + // And persist it locally. db := hctx.GetDb(ctx) err = lib.ReliableDbCreate(db, *entry) @@ -379,11 +381,34 @@ func buildHistoryEntry(ctx context.Context, args []string) (*data.HistoryEntry, entry.EndTime = time.Now().UTC() // command - cmd, err := extractCommandFromArg(ctx, shell, args[4]) - if err != nil { - return nil, err + if shell == "bash" { + cmd, err := getLastCommand(args[4]) + if err != nil { + return nil, fmt.Errorf("failed to build history entry: %w", err) + } + shouldBeSkipped, err := shouldSkipHiddenCommand(ctx, args[4]) + if err != nil { + return nil, fmt.Errorf("failed to check if command was hidden: %w", err) + } + if shouldBeSkipped || strings.HasPrefix(cmd, " ") { + // Don't save commands that start with a space + return nil, nil + } + cmd, err = maybeSkipBashHistTimePrefix(cmd) + if err != nil { + return nil, err + } + entry.Command = cmd + } else if shell == "zsh" || shell == "fish" { + cmd := trimTrailingWhitespace(args[4]) + if strings.HasPrefix(cmd, " ") { + // Don't save commands that start with a space + return nil, nil + } + entry.Command = cmd + } else { + return nil, fmt.Errorf("tried to save a hishtory entry from an unsupported shell=%#v", shell) } - entry.Command = cmd if strings.TrimSpace(entry.Command) == "" { // Skip recording empty commands where the user just hits enter in their terminal return nil, nil @@ -392,38 +417,6 @@ func buildHistoryEntry(ctx context.Context, args []string) (*data.HistoryEntry, return entry, nil } -func extractCommandFromArg(ctx context.Context, shell, arg string) (string, error) { - if shell == "bash" { - cmd, err := getLastCommand(arg) - if err != nil { - return "", fmt.Errorf("failed to build history entry: %w", err) - } - shouldBeSkipped, err := shouldSkipHiddenCommand(ctx, arg) - if err != nil { - return "", fmt.Errorf("failed to check if command was hidden: %w", err) - } - if shouldBeSkipped || strings.HasPrefix(cmd, " ") { - // Don't save commands that start with a space - return "", nil - } - cmd, err = maybeSkipBashHistTimePrefix(cmd) - if err != nil { - return "", err - } - return cmd, nil - } else if shell == "zsh" || shell == "fish" { - cmd := trimTrailingWhitespace(arg) - if strings.HasPrefix(cmd, " ") { - // Don't save commands that start with a space - return "", nil - } - return cmd, nil - } else { - return "", fmt.Errorf("tried to save a hishtory entry from an unsupported shell=%#v", shell) - } - -} - func trimTrailingWhitespace(s string) string { return strings.TrimSuffix(strings.TrimSuffix(s, "\n"), " ") } diff --git a/client/cmd/saveHistoryEntry_test.go b/client/cmd/saveHistoryEntry_test.go index 5d92648..cb76815 100644 --- a/client/cmd/saveHistoryEntry_test.go +++ b/client/cmd/saveHistoryEntry_test.go @@ -10,17 +10,16 @@ import ( "github.com/ddworken/hishtory/client/hctx" "github.com/ddworken/hishtory/client/lib" "github.com/ddworken/hishtory/shared/testutils" - "github.com/stretchr/testify/require" ) func TestBuildHistoryEntry(t *testing.T) { defer testutils.BackupAndRestore(t)() defer testutils.RunTestServer()() - require.NoError(t, lib.Setup("", false)) + testutils.Check(t, lib.Setup("", false)) // Test building an actual entry for bash entry, err := buildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "bash", "120", " 123 ls /foo ", "1641774958"}) - require.NoError(t, err) + testutils.Check(t, err) if entry.ExitCode != 120 { t.Fatalf("history entry has unexpected exit code: %v", entry.ExitCode) } @@ -49,7 +48,7 @@ func TestBuildHistoryEntry(t *testing.T) { // Test building an entry for zsh entry, err = buildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "zsh", "120", "ls /foo\n", "1641774958"}) - require.NoError(t, err) + testutils.Check(t, err) if entry.ExitCode != 120 { t.Fatalf("history entry has unexpected exit code: %v", entry.ExitCode) } @@ -74,7 +73,7 @@ func TestBuildHistoryEntry(t *testing.T) { // Test building an entry for fish entry, err = buildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "fish", "120", "ls /foo\n", "1641774958"}) - require.NoError(t, err) + testutils.Check(t, err) if entry.ExitCode != 120 { t.Fatalf("history entry has unexpected exit code: %v", entry.ExitCode) } @@ -99,7 +98,7 @@ func TestBuildHistoryEntry(t *testing.T) { // Test building an entry that is empty, and thus not saved entry, err = buildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "zsh", "120", " \n", "1641774958"}) - require.NoError(t, err) + testutils.Check(t, err) if entry != nil { t.Fatalf("expected history entry to be nil") } @@ -109,7 +108,7 @@ func TestBuildHistoryEntryWithTimestampStripping(t *testing.T) { defer testutils.BackupAndRestoreEnv("HISTTIMEFORMAT")() defer testutils.BackupAndRestore(t)() defer testutils.RunTestServer()() - require.NoError(t, lib.Setup("", false)) + testutils.Check(t, lib.Setup("", false)) testcases := []struct { input, histtimeformat, expectedCommand string @@ -121,11 +120,11 @@ func TestBuildHistoryEntryWithTimestampStripping(t *testing.T) { for _, tc := range testcases { conf := hctx.GetConf(hctx.MakeContext()) conf.LastSavedHistoryLine = "" - require.NoError(t, hctx.SetConfig(conf)) + testutils.Check(t, hctx.SetConfig(conf)) os.Setenv("HISTTIMEFORMAT", tc.histtimeformat) entry, err := buildHistoryEntry(hctx.MakeContext(), []string{"unused", "saveHistoryEntry", "bash", "120", tc.input, "1641774958"}) - require.NoError(t, err) + testutils.Check(t, err) if entry == nil { t.Fatalf("entry is unexpectedly nil") } @@ -137,12 +136,12 @@ func TestBuildHistoryEntryWithTimestampStripping(t *testing.T) { func TestParseCrossPlatformInt(t *testing.T) { res, err := parseCrossPlatformInt("123") - require.NoError(t, err) + testutils.Check(t, err) if res != 123 { t.Fatalf("failed to parse cross platform int %d", res) } res, err = parseCrossPlatformInt("123N") - require.NoError(t, err) + testutils.Check(t, err) if res != 123 { t.Fatalf("failed to parse cross platform int %d", res) } @@ -178,7 +177,7 @@ func TestGetLastCommand(t *testing.T) { } for _, tc := range testcases { actualOutput, err := getLastCommand(tc.input) - require.NoError(t, err) + testutils.Check(t, err) if actualOutput != tc.expectedOutput { t.Fatalf("getLastCommand(%#v) returned %#v (expected=%#v)", tc.input, actualOutput, tc.expectedOutput) } @@ -220,7 +219,7 @@ func TestMaybeSkipBashHistTimePrefix(t *testing.T) { for _, tc := range testcases { os.Setenv("HISTTIMEFORMAT", tc.env) stripped, err := maybeSkipBashHistTimePrefix(tc.cmdLine) - require.NoError(t, err) + testutils.Check(t, err) if stripped != tc.expected { t.Fatalf("skipping the time prefix returned %#v (expected=%#v for %#v)", stripped, tc.expected, tc.cmdLine) } diff --git a/client/fuzz_test.go b/client/fuzz_test.go index eef048a..ef61289 100644 --- a/client/fuzz_test.go +++ b/client/fuzz_test.go @@ -8,7 +8,6 @@ import ( "github.com/ddworken/hishtory/shared/testutils" "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/require" ) type operation struct { @@ -80,11 +79,11 @@ func fuzzTest(t *testing.T, tester shellTester, input string) { switchToDevice(&devices, op.device) if op.cmd != "" { _, err := tester.RunInteractiveShellRelaxed(t, op.cmd) - require.NoError(t, err) + testutils.Check(t, err) } if op.redactQuery != "" { _, err := tester.RunInteractiveShellRelaxed(t, `HISHTORY_REDACT_FORCE=1 hishtory redact `+op.redactQuery) - require.NoError(t, err) + testutils.Check(t, err) } // Calculate the expected output of hishtory export @@ -112,7 +111,7 @@ func fuzzTest(t *testing.T, tester shellTester, input string) { // Run hishtory export and check the output out, err := tester.RunInteractiveShellRelaxed(t, `hishtory export -export -pipefail`) - require.NoError(t, err) + testutils.Check(t, err) expectedOutput := keyToCommands[op.device.key] if diff := cmp.Diff(expectedOutput, out); diff != "" { t.Fatalf("hishtory export mismatch for input=%#v key=%s (-expected +got):\n%s\nout=%#v", input, op.device.key, diff, out) @@ -123,7 +122,7 @@ func fuzzTest(t *testing.T, tester shellTester, input string) { for _, op := range ops { switchToDevice(&devices, op.device) out, err := tester.RunInteractiveShellRelaxed(t, `hishtory export -export -pipefail`) - require.NoError(t, err) + testutils.Check(t, err) expectedOutput := keyToCommands[op.device.key] if diff := cmp.Diff(expectedOutput, out); diff != "" { t.Fatalf("hishtory export mismatch for key=%s (-expected +got):\n%s\nout=%#v", op.device.key, diff, out) diff --git a/client/lib/config.fish b/client/lib/config.fish index 0cb0d78..a0e6cb9 100644 --- a/client/lib/config.fish +++ b/client/lib/config.fish @@ -2,8 +2,8 @@ function _hishtory_post_exec --on-event fish_preexec # Runs after , but before the command is executed set --global _hishtory_command $argv set --global _hishtory_start_time (date +%s) - hishtory presaveHistoryEntry fish "$_hishtory_command" $_hishtory_start_time & # Background Run - # hishtory presaveHistoryEntry fish "$_hishtory_command" $_hishtory_start_time # Foreground Run + hishtory presaveHistoryEntry bash "$_hishtory_command" $_hishtory_start_time & # Background Run + # hishtory presaveHistoryEntry bash "$_hishtory_command" $_hishtory_start_time # Foreground Run end set --global _hishtory_first_prompt 1 diff --git a/client/lib/config.sh b/client/lib/config.sh index 7005ecd..bb90639 100644 --- a/client/lib/config.sh +++ b/client/lib/config.sh @@ -14,10 +14,9 @@ function __hishtory_precommand() { # Run before every command HISHTORY_START_TIME=`date +%s` - CMD=`history 1` - if ! [ -z "CMD " ] ; then - (hishtory presaveHistoryEntry bash "$CMD" $HISHTORY_START_TIME &) # Background Run - # hishtory presaveHistoryEntry bash "$CMD" $HISHTORY_START_TIME # Foreground Run + if ! [ -z "BASH_COMMAND " ] && [ "$BASH_COMMAND" != "__hishtory_postcommand" ]; then + (hishtory presaveHistoryEntry bash "$BASH_COMMAND" $HISHTORY_START_TIME &) # Background Run + # hishtory presaveHistoryEntry bash "$BASH_COMMAND" $HISHTORY_START_TIME # Foreground Run fi } trap "__hishtory_precommand" DEBUG diff --git a/client/lib/goldens/testRemoveDuplicateRows-enabled-tquery b/client/lib/goldens/testRemoveDuplicateRows-enabled-tquery index ab861cd..c65a7b0 100644 --- a/client/lib/goldens/testRemoveDuplicateRows-enabled-tquery +++ b/client/lib/goldens/testRemoveDuplicateRows-enabled-tquery @@ -1,3 +1,7 @@ +-pipefail + + + Search Query: > -pipefail ┌───────────────────────────────────────────────────────────────────────────┐ diff --git a/client/lib/lib.go b/client/lib/lib.go index 1135f84..3721c58 100644 --- a/client/lib/lib.go +++ b/client/lib/lib.go @@ -170,21 +170,6 @@ func BuildTableRow(ctx context.Context, columnNames []string, entry data.History return row, nil } -// Make a regex that matches the non-tokenized bits of the given query -func MakeRegexFromQuery(query string) string { - tokens := tokenize(strings.TrimSpace(query)) - r := "" - for _, token := range tokens { - if !strings.HasPrefix(token, "-") && !containsUnescaped(token, ":") { - if r != "" { - r += "|" - } - r += fmt.Sprintf("(%s)", regexp.QuoteMeta(token)) - } - } - return r -} - func stringArrayToAnyArray(arr []string) []any { ret := make([]any, 0) for _, item := range arr { @@ -726,7 +711,10 @@ func where(tx *gorm.DB, s string, v1 any, v2 any) *gorm.DB { } func MakeWhereQueryFromSearch(ctx context.Context, db *gorm.DB, query string) (*gorm.DB, error) { - tokens := tokenize(query) + tokens, err := tokenize(query) + if err != nil { + return nil, fmt.Errorf("failed to tokenize query: %w", err) + } tx := db.Model(&data.HistoryEntry{}).Where("true") for _, token := range tokens { if strings.HasPrefix(token, "-") { @@ -903,11 +891,11 @@ func getAllCustomColumnNames(ctx context.Context) ([]string, error) { return ccNames, nil } -func tokenize(query string) []string { +func tokenize(query string) ([]string, error) { if query == "" { - return []string{} + return []string{}, nil } - return splitEscaped(query, ' ', -1) + return splitEscaped(query, ' ', -1), nil } func splitEscaped(query string, separator rune, maxSplit int) []string { diff --git a/client/lib/lib_test.go b/client/lib/lib_test.go index fe4e664..9930bef 100644 --- a/client/lib/lib_test.go +++ b/client/lib/lib_test.go @@ -26,16 +26,16 @@ func TestSetup(t *testing.T) { defer testutils.RunTestServer()() homedir, err := os.UserHomeDir() - require.NoError(t, err) + testutils.Check(t, err) if _, err := os.Stat(path.Join(homedir, data.GetHishtoryPath(), data.CONFIG_PATH)); err == nil { t.Fatalf("hishtory secret file already exists!") } - require.NoError(t, Setup("", false)) + testutils.Check(t, Setup("", false)) if _, err := os.Stat(path.Join(homedir, data.GetHishtoryPath(), data.CONFIG_PATH)); err != nil { t.Fatalf("hishtory secret file does not exist after Setup()!") } data, err := os.ReadFile(path.Join(homedir, data.GetHishtoryPath(), data.CONFIG_PATH)) - require.NoError(t, err) + testutils.Check(t, err) if len(data) < 10 { t.Fatalf("hishtory secret has unexpected length: %d", len(data)) } @@ -50,16 +50,16 @@ func TestSetupOffline(t *testing.T) { defer testutils.RunTestServer()() homedir, err := os.UserHomeDir() - require.NoError(t, err) + testutils.Check(t, err) if _, err := os.Stat(path.Join(homedir, data.GetHishtoryPath(), data.CONFIG_PATH)); err == nil { t.Fatalf("hishtory secret file already exists!") } - require.NoError(t, Setup("", true)) + testutils.Check(t, Setup("", true)) if _, err := os.Stat(path.Join(homedir, data.GetHishtoryPath(), data.CONFIG_PATH)); err != nil { t.Fatalf("hishtory secret file does not exist after Setup()!") } data, err := os.ReadFile(path.Join(homedir, data.GetHishtoryPath(), data.CONFIG_PATH)) - require.NoError(t, err) + testutils.Check(t, err) if len(data) < 10 { t.Fatalf("hishtory secret has unexpected length: %d", len(data)) } @@ -70,14 +70,14 @@ func TestSetupOffline(t *testing.T) { } func TestPersist(t *testing.T) { defer testutils.BackupAndRestore(t)() - require.NoError(t, hctx.InitConfig()) + testutils.Check(t, hctx.InitConfig()) db := hctx.GetDb(hctx.MakeContext()) entry := testutils.MakeFakeHistoryEntry("ls ~/") - require.NoError(t, db.Create(entry).Error) + testutils.Check(t, db.Create(entry).Error) var historyEntries []*data.HistoryEntry result := db.Find(&historyEntries) - require.NoError(t, result.Error) + testutils.Check(t, result.Error) if len(historyEntries) != 1 { t.Fatalf("DB has %d entries, expected 1!", len(historyEntries)) } @@ -89,19 +89,19 @@ func TestPersist(t *testing.T) { func TestSearch(t *testing.T) { defer testutils.BackupAndRestore(t)() - require.NoError(t, hctx.InitConfig()) + testutils.Check(t, hctx.InitConfig()) ctx := hctx.MakeContext() db := hctx.GetDb(ctx) // Insert data entry1 := testutils.MakeFakeHistoryEntry("ls /foo") - require.NoError(t, db.Create(entry1).Error) + testutils.Check(t, db.Create(entry1).Error) entry2 := testutils.MakeFakeHistoryEntry("ls /bar") - require.NoError(t, db.Create(entry2).Error) + testutils.Check(t, db.Create(entry2).Error) // Search for data results, err := Search(ctx, db, "ls", 5) - require.NoError(t, err) + testutils.Check(t, err) if len(results) != 2 { t.Fatalf("Search() returned %d results, expected 2, results=%#v", len(results), results) } @@ -114,61 +114,61 @@ func TestSearch(t *testing.T) { // Search but exclude bar results, err = Search(ctx, db, "ls -bar", 5) - require.NoError(t, err) + testutils.Check(t, err) if len(results) != 1 { t.Fatalf("Search() returned %d results, expected 1, results=%#v", len(results), results) } // Search but exclude foo results, err = Search(ctx, db, "ls -foo", 5) - require.NoError(t, err) + testutils.Check(t, err) if len(results) != 1 { t.Fatalf("Search() returned %d results, expected 1, results=%#v", len(results), results) } // Search but include / also results, err = Search(ctx, db, "ls /", 5) - require.NoError(t, err) + testutils.Check(t, err) if len(results) != 2 { t.Fatalf("Search() returned %d results, expected 1, results=%#v", len(results), results) } // Search but exclude slash results, err = Search(ctx, db, "ls -/", 5) - require.NoError(t, err) + testutils.Check(t, err) if len(results) != 0 { t.Fatalf("Search() returned %d results, expected 0, results=%#v", len(results), results) } // Tests for escaping - require.NoError(t, db.Create(testutils.MakeFakeHistoryEntry("ls -baz")).Error) + testutils.Check(t, db.Create(testutils.MakeFakeHistoryEntry("ls -baz")).Error) results, err = Search(ctx, db, "ls", 5) - require.NoError(t, err) + testutils.Check(t, err) if len(results) != 3 { t.Fatalf("Search() returned %d results, expected 3, results=%#v", len(results), results) } results, err = Search(ctx, db, "ls -baz", 5) - require.NoError(t, err) + testutils.Check(t, err) if len(results) != 2 { t.Fatalf("Search() returned %d results, expected 2, results=%#v", len(results), results) } results, err = Search(ctx, db, "ls \\-baz", 5) - require.NoError(t, err) + testutils.Check(t, err) if len(results) != 1 { t.Fatalf("Search() returned %d results, expected 1, results=%#v", len(results), results) } // A malformed search query, but we should just ignore the dash since this is a common enough thing results, err = Search(ctx, db, "ls -", 5) - require.NoError(t, err) + testutils.Check(t, err) if len(results) != 3 { t.Fatalf("Search() returned %d results, expected 3, results=%#v", len(results), results) } // A search for an entry containing a backslash - require.NoError(t, db.Create(testutils.MakeFakeHistoryEntry("echo '\\'")).Error) + testutils.Check(t, db.Create(testutils.MakeFakeHistoryEntry("echo '\\'")).Error) results, err = Search(ctx, db, "\\\\", 5) - require.NoError(t, err) + testutils.Check(t, err) if len(results) != 1 { t.Fatalf("Search() returned %d results, expected 3, results=%#v", len(results), results) } @@ -177,7 +177,7 @@ func TestSearch(t *testing.T) { func TestAddToDbIfNew(t *testing.T) { // Set up defer testutils.BackupAndRestore(t)() - require.NoError(t, hctx.InitConfig()) + testutils.Check(t, hctx.InitConfig()) db := hctx.GetDb(hctx.MakeContext()) // Add duplicate entries @@ -239,52 +239,52 @@ func TestZshWeirdness(t *testing.T) { func TestParseTimeGenerously(t *testing.T) { ts, err := parseTimeGenerously("2006-01-02T15:04:00-08:00") - require.NoError(t, err) + testutils.Check(t, err) if ts.Unix() != 1136243040 { t.Fatalf("parsed time incorrectly: %d", ts.Unix()) } ts, err = parseTimeGenerously("2006-01-02 T15:04:00 -08:00") - require.NoError(t, err) + testutils.Check(t, err) if ts.Unix() != 1136243040 { t.Fatalf("parsed time incorrectly: %d", ts.Unix()) } ts, err = parseTimeGenerously("2006-01-02_T15:04:00_-08:00") - require.NoError(t, err) + testutils.Check(t, err) if ts.Unix() != 1136243040 { t.Fatalf("parsed time incorrectly: %d", ts.Unix()) } ts, err = parseTimeGenerously("2006-01-02T15:04:00") - require.NoError(t, err) + testutils.Check(t, err) if ts.Year() != 2006 || ts.Month() != time.January || ts.Day() != 2 || ts.Hour() != 15 || ts.Minute() != 4 || ts.Second() != 0 { t.Fatalf("parsed time incorrectly: %d", ts.Unix()) } ts, err = parseTimeGenerously("2006-01-02_T15:04:00") - require.NoError(t, err) + testutils.Check(t, err) if ts.Year() != 2006 || ts.Month() != time.January || ts.Day() != 2 || ts.Hour() != 15 || ts.Minute() != 4 || ts.Second() != 0 { t.Fatalf("parsed time incorrectly: %d", ts.Unix()) } ts, err = parseTimeGenerously("2006-01-02_15:04:00") - require.NoError(t, err) + testutils.Check(t, err) if ts.Year() != 2006 || ts.Month() != time.January || ts.Day() != 2 || ts.Hour() != 15 || ts.Minute() != 4 || ts.Second() != 0 { t.Fatalf("parsed time incorrectly: %d", ts.Unix()) } ts, err = parseTimeGenerously("2006-01-02T15:04") - require.NoError(t, err) + testutils.Check(t, err) if ts.Year() != 2006 || ts.Month() != time.January || ts.Day() != 2 || ts.Hour() != 15 || ts.Minute() != 4 || ts.Second() != 0 { t.Fatalf("parsed time incorrectly: %d", ts.Unix()) } ts, err = parseTimeGenerously("2006-01-02_15:04") - require.NoError(t, err) + testutils.Check(t, err) if ts.Year() != 2006 || ts.Month() != time.January || ts.Day() != 2 || ts.Hour() != 15 || ts.Minute() != 4 || ts.Second() != 0 { t.Fatalf("parsed time incorrectly: %d", ts.Unix()) } ts, err = parseTimeGenerously("2006-01-02") - require.NoError(t, err) + testutils.Check(t, err) if ts.Year() != 2006 || ts.Month() != time.January || ts.Day() != 2 || ts.Hour() != 0 || ts.Minute() != 0 || ts.Second() != 0 { t.Fatalf("parsed time incorrectly: %d", ts.Unix()) } ts, err = parseTimeGenerously("1693163976") - require.NoError(t, err) + testutils.Check(t, err) if ts.Year() != 2023 || ts.Month() != time.August || ts.Day() != 27 || ts.Hour() != 12 || ts.Minute() != 19 || ts.Second() != 36 { t.Fatalf("parsed time incorrectly: %d %s", ts.Unix(), ts.GoString()) } diff --git a/client/table/table.go b/client/table/table.go index 5d3579f..06fa12e 100644 --- a/client/table/table.go +++ b/client/table/table.go @@ -1,5 +1,4 @@ // Forked from https://github.com/charmbracelet/bubbles/blob/master/table/table.go to add horizontal scrolling -// Also includes https://github.com/charmbracelet/bubbles/pull/397/files to support cell styling package table @@ -32,13 +31,6 @@ type Model struct { hcursor int } -// CellPosition holds row and column indexes. -type CellPosition struct { - RowID int - Column int - IsRowSelected bool -} - // Row represents one line in the table. type Row []string @@ -116,32 +108,6 @@ type Styles struct { Header lipgloss.Style Cell lipgloss.Style Selected lipgloss.Style - - // RenderCell is a low-level primitive for stylizing cells. - // It is responsible for rendering the selection style. Styles.Cell is ignored. - // - // Example implementation: - // s.RenderCell = func(model table.Model, value string, position table.CellPosition) string { - // cellStyle := s.Cell.Copy() - // - // switch { - // case position.IsRowSelected: - // return cellStyle.Background(lipgloss.Color("57")).Render(value) - // case position.Column == 1: - // return cellStyle.Foreground(lipgloss.Color("21")).Render(value) - // default: - // return cellStyle.Render(value) - // } - // } - RenderCell func(model Model, value string, position CellPosition) string -} - -func (s Styles) renderCell(model Model, value string, position CellPosition) string { - if s.RenderCell != nil { - return s.RenderCell(model, value, position) - } - - return s.Cell.Render(value) } // DefaultStyles returns a set of default style definitions for this table. @@ -479,30 +445,21 @@ func (m Model) headersView() string { } func (m *Model) renderRow(rowID int) string { - isRowSelected := rowID == m.cursor var s = make([]string, 0, len(m.cols)) for i, value := range m.rows[rowID] { style := lipgloss.NewStyle().Width(m.cols[i].Width).MaxWidth(m.cols[i].Width).Inline(true) - - position := CellPosition{ - RowID: rowID, - Column: i, - IsRowSelected: isRowSelected, - } - var renderedCell string if i == m.ColIndex(m.hcol) && m.hcursor > 0 { - renderedCell = style.Render(runewidth.Truncate(runewidth.TruncateLeft(value, m.hcursor, "…"), m.cols[i].Width, "…")) + renderedCell = m.styles.Cell.Render(style.Render(runewidth.Truncate(runewidth.TruncateLeft(value, m.hcursor, "…"), m.cols[i].Width, "…"))) } else { - renderedCell = style.Render(runewidth.Truncate(value, m.cols[i].Width, "…")) + renderedCell = m.styles.Cell.Render(style.Render(runewidth.Truncate(value, m.cols[i].Width, "…"))) } - renderedCell = m.styles.renderCell(*m, renderedCell, position) s = append(s, renderedCell) } row := lipgloss.JoinHorizontal(lipgloss.Left, s...) - if isRowSelected { + if rowID == m.cursor { return m.styles.Selected.Render(row) } diff --git a/client/testutils.go b/client/testutils.go index 89517f7..d53a544 100644 --- a/client/testutils.go +++ b/client/testutils.go @@ -198,15 +198,15 @@ func hishtoryQuery(t testing.TB, tester shellTester, query string) string { func manuallySubmitHistoryEntry(t testing.TB, userSecret string, entry data.HistoryEntry) { encEntry, err := data.EncryptHistoryEntry(userSecret, entry) - require.NoError(t, err) + testutils.Check(t, err) if encEntry.Date != entry.EndTime { t.Fatalf("encEntry.Date does not match the entry") } jsonValue, err := json.Marshal([]shared.EncHistoryEntry{encEntry}) - require.NoError(t, err) + testutils.Check(t, err) require.NotEqual(t, "", entry.DeviceId) resp, err := http.Post("http://localhost:8080/api/v1/submit?source_device_id="+entry.DeviceId, "application/json", bytes.NewBuffer(jsonValue)) - require.NoError(t, err) + testutils.Check(t, err) if resp.StatusCode != 200 { t.Fatalf("failed to submit result to backend, status_code=%d", resp.StatusCode) } @@ -246,41 +246,11 @@ func captureTerminalOutputWithShellName(t testing.TB, tester shellTester, overri } func captureTerminalOutputWithShellNameAndDimensions(t testing.TB, tester shellTester, overriddenShellName string, width, height int, commands []TmuxCommand) string { - return captureTerminalOutputComplex(t, - TmuxCaptureConfig{ - tester: tester, - overriddenShellName: overriddenShellName, - width: width, - height: height, - complexCommands: commands, - }) -} - -type TmuxCaptureConfig struct { - tester shellTester - overriddenShellName string - commands []string - complexCommands []TmuxCommand - width, height int - includeEscapeSequences bool -} - -func captureTerminalOutputComplex(t testing.TB, captureConfig TmuxCaptureConfig) string { - require.NotNil(t, captureConfig.tester) - if captureConfig.overriddenShellName == "" { - captureConfig.overriddenShellName = captureConfig.tester.ShellName() - } - if captureConfig.width == 0 { - captureConfig.width = 200 - } - if captureConfig.height == 0 { - captureConfig.height = 50 - } sleepAmount := "0.1" if runtime.GOOS == "linux" { sleepAmount = "0.2" } - if captureConfig.overriddenShellName == "fish" { + if overriddenShellName == "fish" { // Fish is considerably slower so this is sadly necessary sleepAmount = "0.5" } @@ -289,20 +259,13 @@ func captureTerminalOutputComplex(t testing.TB, captureConfig TmuxCaptureConfig) } fullCommand := "" fullCommand += " tmux kill-session -t foo || true\n" - fullCommand += fmt.Sprintf(" tmux -u new-session -d -x %d -y %d -s foo %s\n", captureConfig.width, captureConfig.height, captureConfig.overriddenShellName) + fullCommand += fmt.Sprintf(" tmux -u new-session -d -x %d -y %d -s foo %s\n", width, height, overriddenShellName) fullCommand += " sleep 1\n" - if captureConfig.overriddenShellName == "bash" { + if overriddenShellName == "bash" { fullCommand += " tmux send -t foo SPACE source SPACE ~/.bashrc ENTER\n" } fullCommand += " sleep " + sleepAmount + "\n" - if len(captureConfig.commands) > 0 { - require.Empty(t, captureConfig.complexCommands) - for _, command := range captureConfig.commands { - captureConfig.complexCommands = append(captureConfig.complexCommands, TmuxCommand{Keys: command}) - } - } - require.NotEmpty(t, captureConfig.complexCommands) - for _, cmd := range captureConfig.complexCommands { + for _, cmd := range commands { if cmd.Keys != "" { fullCommand += " tmux send -t foo -- " fullCommand += cmd.Keys @@ -320,21 +283,17 @@ func captureTerminalOutputComplex(t testing.TB, captureConfig TmuxCaptureConfig) if testutils.IsGithubAction() { fullCommand += " sleep 2.5\n" } - fullCommand += " tmux capture-pane -t foo -p" - if captureConfig.includeEscapeSequences { - fullCommand += "e" - } - fullCommand += "\n" + fullCommand += " tmux capture-pane -t foo -p\n" fullCommand += " tmux kill-session -t foo\n" testutils.TestLog(t, "Running tmux command: "+fullCommand) - return strings.TrimSpace(captureConfig.tester.RunInteractiveShell(t, fullCommand)) + return strings.TrimSpace(tester.RunInteractiveShell(t, fullCommand)) } func assertNoLeakedConnections(t testing.TB) { resp, err := lib.ApiGet("/api/v1/get-num-connections") - require.NoError(t, err) + testutils.Check(t, err) numConnections, err := strconv.Atoi(string(resp)) - require.NoError(t, err) + testutils.Check(t, err) if numConnections > 1 { t.Fatalf("DB has %d open connections, expected to have 1 or less", numConnections) } diff --git a/client/tui/tui.go b/client/tui/tui.go index 49db438..443d9f4 100644 --- a/client/tui/tui.go +++ b/client/tui/tui.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "regexp" "strings" "time" @@ -28,7 +27,6 @@ import ( const TABLE_HEIGHT = 20 const PADDED_NUM_ENTRIES = TABLE_HEIGHT * 5 -var CURRENT_QUERY_FOR_HIGHLIGHTING string = "" var SELECTED_COMMAND string = "" var baseStyle = lipgloss.NewStyle(). @@ -209,7 +207,6 @@ func initialModel(ctx context.Context, initialQuery string) model { if initialQuery != "" { queryInput.SetValue(initialQuery) } - CURRENT_QUERY_FOR_HIGHLIGHTING = initialQuery return model{ctx: ctx, spinner: s, isLoading: true, table: nil, tableEntries: []*data.HistoryEntry{}, runQuery: &initialQuery, queryInput: queryInput, help: help.New()} } @@ -322,7 +319,6 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.queryInput = i searchQuery := m.queryInput.Value() m.runQuery = &searchQuery - CURRENT_QUERY_FOR_HIGHLIGHTING = searchQuery cmd3 := runQueryAndUpdateTable(m, false, false) preventTableOverscrolling(m) return m, tea.Batch(pendingCommands, cmd2, cmd3) @@ -583,73 +579,6 @@ func makeTable(ctx context.Context, rows []table.Row) (table.Model, error) { Foreground(lipgloss.Color("229")). Background(lipgloss.Color("57")). Bold(false) - if config.BetaMode { - MATCH_NOTHING_REGEXP := regexp.MustCompile("a^") - s.RenderCell = func(model table.Model, value string, position table.CellPosition) string { - var re *regexp.Regexp - CURRENT_QUERY_FOR_HIGHLIGHTING = strings.TrimSpace(CURRENT_QUERY_FOR_HIGHLIGHTING) - if CURRENT_QUERY_FOR_HIGHLIGHTING == "" { - // If there is no search query, then there is nothing to highlight - re = MATCH_NOTHING_REGEXP - } else { - queryRegex := lib.MakeRegexFromQuery(CURRENT_QUERY_FOR_HIGHLIGHTING) - r, err := regexp.Compile(queryRegex) - if err != nil { - // Failed to compile the regex for highlighting matches, this should never happen. In this - // case, just use a regexp that matches nothing to ensure that the TUI doesn't crash. - re = MATCH_NOTHING_REGEXP - } else { - re = r - } - } - - // func to render a given chunk of `value`. `isMatching` is whether `v` matches the search query (and - // thus needs to be highlighted). `isLeftMost` and `isRightMost` determines whether additional - // padding is added (to reproduce the padding that `s.Cell` normally adds). - renderChunk := func(v string, isMatching, isLeftMost, isRightMost bool) string { - chunkStyle := lipgloss.NewStyle() - if position.IsRowSelected { - // Apply the selected style as the base style if this is the highlighted row of the table - chunkStyle = s.Selected.Copy() - } - if isLeftMost { - chunkStyle = chunkStyle.PaddingLeft(1) - } - if isRightMost { - chunkStyle = chunkStyle.PaddingRight(1) - } - if isMatching { - chunkStyle = chunkStyle.Bold(true) - } - return chunkStyle.Render(v) - } - - matches := re.FindAllStringIndex(value, -1) - if len(matches) == 0 { - // No matches, so render the entire value - return renderChunk(value /*isMatching = */, false /*isLeftMost = */, true /*isRightMost = */, true) - } - - // Iterate through the chunks of the value and highlight the relevant pieces - ret := "" - lastIncludedIdx := 0 - for _, match := range re.FindAllStringIndex(value, -1) { - matchStartIdx := match[0] - matchEndIdx := match[1] - beforeMatch := value[lastIncludedIdx:matchStartIdx] - if beforeMatch != "" { - ret += renderChunk(beforeMatch, false, lastIncludedIdx == 0, false) - } - match := value[matchStartIdx:matchEndIdx] - ret += renderChunk(match, true, matchStartIdx == 0, matchEndIdx == len(value)) - lastIncludedIdx = matchEndIdx - } - if lastIncludedIdx != len(value) { - ret += renderChunk(value[lastIncludedIdx:], false, false, true) - } - return ret - } - } t.SetStyles(s) t.Focus() return t, nil diff --git a/shared/testutils/testutils.go b/shared/testutils/testutils.go index 5aad3ca..cc4be11 100644 --- a/shared/testutils/testutils.go +++ b/shared/testutils/testutils.go @@ -18,7 +18,6 @@ import ( "github.com/ddworken/hishtory/client/data" "github.com/google/go-cmp/cmp" "github.com/google/uuid" - "github.com/stretchr/testify/require" ) const ( @@ -50,7 +49,7 @@ func getInitialWd() string { func ResetLocalState(t *testing.T) { homedir, err := os.UserHomeDir() - require.NoError(t, err) + Check(t, err) persistLog() _ = BackupAndRestoreWithId(t, "-reset-local-state") _ = os.RemoveAll(path.Join(homedir, data.GetHishtoryPath())) @@ -70,10 +69,10 @@ func getBackPath(file, id string) string { func BackupAndRestoreWithId(t testing.TB, id string) func() { ResetFakeHistoryTimestamp() homedir, err := os.UserHomeDir() - require.NoError(t, err) + Check(t, err) initialWd, err := os.Getwd() - require.NoError(t, err) - require.NoError(t, os.MkdirAll(path.Join(homedir, data.GetHishtoryPath()+".test"), os.ModePerm)) + Check(t, err) + Check(t, os.MkdirAll(path.Join(homedir, data.GetHishtoryPath()+".test"), os.ModePerm)) renameFiles := []string{ path.Join(homedir, data.GetHishtoryPath(), data.DB_PATH), @@ -90,7 +89,7 @@ func BackupAndRestoreWithId(t testing.TB, id string) func() { } for _, file := range renameFiles { touchFile(file) - require.NoError(t, os.Rename(file, getBackPath(file, id))) + Check(t, os.Rename(file, getBackPath(file, id))) } copyFiles := []string{ path.Join(homedir, ".zshrc"), @@ -99,7 +98,7 @@ func BackupAndRestoreWithId(t testing.TB, id string) func() { } for _, file := range copyFiles { touchFile(file) - require.NoError(t, copy(file, getBackPath(file, id))) + Check(t, copy(file, getBackPath(file, id))) } configureZshrc(homedir) touchFile(path.Join(homedir, ".bash_history")) @@ -112,8 +111,8 @@ func BackupAndRestoreWithId(t testing.TB, id string) func() { t.Fatalf("failed to execute killall hishtory, stdout=%#v: %v", string(stdout), err) } persistLog() - require.NoError(t, os.RemoveAll(path.Join(homedir, data.GetHishtoryPath()))) - require.NoError(t, os.MkdirAll(path.Join(homedir, data.GetHishtoryPath()), os.ModePerm)) + Check(t, os.RemoveAll(path.Join(homedir, data.GetHishtoryPath()))) + Check(t, os.MkdirAll(path.Join(homedir, data.GetHishtoryPath()), os.ModePerm)) for _, file := range renameFiles { checkError(os.Rename(getBackPath(file, id), file)) } @@ -291,6 +290,20 @@ func RunTestServer() func() { } } +func Check(t testing.TB, err error) { + if err != nil { + _, filename, line, _ := runtime.Caller(1) + t.Fatalf("Unexpected error at %s:%d: %v", filename, line, err) + } +} + +func CheckWithInfo(t *testing.T, err error, additionalInfo string) { + if err != nil { + _, filename, line, _ := runtime.Caller(1) + t.Fatalf("Unexpected error: %v at %s:%d! Additional info: %v", err, filename, line, additionalInfo) + } +} + func IsOnline() bool { _, err := http.Get("https://hishtory.dev") return err == nil @@ -325,12 +338,12 @@ func IsGithubAction() bool { func TestLog(t testing.TB, line string) { f, err := os.OpenFile("/tmp/test.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) if err != nil { - require.NoError(t, err) + Check(t, err) } defer f.Close() _, err = f.WriteString(line + "\n") if err != nil { - require.NoError(t, err) + Check(t, err) } } @@ -359,7 +372,7 @@ func CompareGoldens(t testing.TB, out, goldenName string) { if os.IsNotExist(err) { expected = []byte("ERR_FILE_NOT_FOUND:" + goldenPath) } else { - require.NoError(t, err) + Check(t, err) } } if diff := cmp.Diff(string(expected), out); diff != "" { @@ -367,7 +380,7 @@ func CompareGoldens(t testing.TB, out, goldenName string) { _, filename, line, _ := runtime.Caller(1) t.Fatalf("hishtory golden mismatch for %s at %s:%d (-expected +got):\n%s\nactual=\n%s", goldenName, filename, line, diff, out) } else { - require.NoError(t, os.WriteFile(goldenPath, []byte(out), 0644)) + Check(t, os.WriteFile(goldenPath, []byte(out), 0644)) } } }