mirror of
https://github.com/ddworken/hishtory.git
synced 2024-11-22 08:14:02 +01:00
Add progress bar to monitor importing history entries for #93
This commit is contained in:
David Dworken
parent
851283df8d
commit
dbb555e20c
@ -9,3 +9,4 @@ os.Unsetenv
|
||||
(*github.com/DataDog/datadog-go/statsd.Client).Count
|
||||
(*github.com/DataDog/datadog-go/statsd.Client).Incr
|
||||
(*github.com/DataDog/datadog-go/statsd.Client).Distribution
|
||||
(*github.com/schollz/progressbar/v3.ProgressBar).Finish
|
||||
|
||||
@ -18,16 +18,19 @@ import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
_ "embed" // for embedding config.sh
|
||||
|
||||
"golang.org/x/exp/slices"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/araddon/dateparse"
|
||||
"github.com/fatih/color"
|
||||
"github.com/google/uuid"
|
||||
"github.com/rodaine/table"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
|
||||
"github.com/ddworken/hishtory/client/data"
|
||||
"github.com/ddworken/hishtory/client/hctx"
|
||||
@ -258,6 +261,48 @@ func isBashWeirdness(cmd string) bool {
|
||||
return BASH_FIRST_COMMAND_BUG_REGEX.MatchString(cmd)
|
||||
}
|
||||
|
||||
func countLinesInFile(filename string) (int, error) {
|
||||
if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
|
||||
return 0, nil
|
||||
}
|
||||
file, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
buf := make([]byte, 32*1024)
|
||||
count := 0
|
||||
lineSep := []byte{'\n'}
|
||||
|
||||
for {
|
||||
c, err := file.Read(buf)
|
||||
count += bytes.Count(buf[:c], lineSep)
|
||||
|
||||
switch {
|
||||
case err == io.EOF:
|
||||
return count, nil
|
||||
|
||||
case err != nil:
|
||||
return count, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func countLinesInFiles(filenames ...string) (int, error) {
|
||||
total := 0
|
||||
for _, f := range filenames {
|
||||
l, err := countLinesInFile(f)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
total += l
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// The number of entries where if we're importing more than this many entries, the import is likely to be
|
||||
// slow, and it is then worth displaying a progress bar.
|
||||
const NUM_IMPORTED_ENTRIES_SLOW int = 20_000
|
||||
|
||||
func ImportHistory(ctx context.Context, shouldReadStdin, force bool) (int, error) {
|
||||
config := hctx.GetConf(ctx)
|
||||
if config.HaveCompletedInitialImport && !force {
|
||||
@ -265,15 +310,24 @@ func ImportHistory(ctx context.Context, shouldReadStdin, force bool) (int, error
|
||||
return 0, nil
|
||||
}
|
||||
homedir := hctx.GetHome(ctx)
|
||||
bashHistPath := filepath.Join(homedir, ".bash_history")
|
||||
zshHistPath := filepath.Join(homedir, ".zsh_history")
|
||||
entriesIter := concatIterators(readFileToIterator(bashHistPath), readFileToIterator(zshHistPath), parseFishHistory(homedir))
|
||||
if histfile := os.Getenv("HISTFILE"); histfile != "" && histfile != zshHistPath && histfile != bashHistPath {
|
||||
entriesIter = concatIterators(entriesIter, readFileToIterator(histfile))
|
||||
inputFiles := []string{
|
||||
filepath.Join(homedir, ".bash_history"),
|
||||
filepath.Join(homedir, ".zsh_history"),
|
||||
}
|
||||
if histfile := os.Getenv("HISTFILE"); histfile != "" && !slices.Contains[string](inputFiles, histfile) {
|
||||
inputFiles = append(inputFiles, histfile)
|
||||
}
|
||||
zHistPath := filepath.Join(homedir, ".zhistory")
|
||||
if zHistPath != os.Getenv("HISTFILE") {
|
||||
entriesIter = concatIterators(entriesIter, readFileToIterator(zHistPath))
|
||||
if !slices.Contains(inputFiles, zHistPath) {
|
||||
inputFiles = append(inputFiles, zHistPath)
|
||||
}
|
||||
entriesIter := parseFishHistory(homedir)
|
||||
for _, file := range inputFiles {
|
||||
entriesIter = concatIterators(entriesIter, readFileToIterator(file))
|
||||
}
|
||||
totalNumEntries, err := countLinesInFiles(inputFiles...)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count input lines during hishtory import: %w", err)
|
||||
}
|
||||
if shouldReadStdin {
|
||||
extraEntries, err := readStdin()
|
||||
@ -281,7 +335,13 @@ func ImportHistory(ctx context.Context, shouldReadStdin, force bool) (int, error
|
||||
return 0, fmt.Errorf("failed to read stdin: %w", err)
|
||||
}
|
||||
entriesIter = concatIterators(entriesIter, Values(extraEntries))
|
||||
totalNumEntries += len(extraEntries)
|
||||
}
|
||||
fishLines, err := countLinesInFile(getFishHistoryPath(homedir))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count fish history lines during hishtory import: %w", err)
|
||||
}
|
||||
totalNumEntries += fishLines
|
||||
db := hctx.GetDb(ctx)
|
||||
currentUser, err := user.Current()
|
||||
if err != nil {
|
||||
@ -297,6 +357,12 @@ func ImportHistory(ctx context.Context, shouldReadStdin, force bool) (int, error
|
||||
importTimestamp := time.Now().UTC()
|
||||
batchSize := 100
|
||||
importEntryId := uuid.Must(uuid.NewRandom()).String()
|
||||
var bar *progressbar.ProgressBar
|
||||
if totalNumEntries > NUM_IMPORTED_ENTRIES_SLOW {
|
||||
fmt.Println("Importing existing history entries")
|
||||
bar = progressbar.Default(int64(totalNumEntries))
|
||||
defer bar.Finish()
|
||||
}
|
||||
entriesIter(func(cmd string, err error) bool {
|
||||
if err != nil {
|
||||
iteratorError = err
|
||||
@ -339,6 +405,12 @@ func ImportHistory(ctx context.Context, shouldReadStdin, force bool) (int, error
|
||||
batch = make([]data.HistoryEntry, 0)
|
||||
}
|
||||
numEntriesImported += 1
|
||||
if bar != nil {
|
||||
_ = bar.Add(1)
|
||||
if numEntriesImported > totalNumEntries {
|
||||
bar.ChangeMax(-1)
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if iteratorError != nil {
|
||||
@ -389,8 +461,12 @@ func readStdin() ([]string, error) {
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func getFishHistoryPath(homedir string) string {
|
||||
return filepath.Join(homedir, ".local/share/fish/fish_history")
|
||||
}
|
||||
|
||||
func parseFishHistory(homedir string) Seq2[string, error] {
|
||||
lines := readFileToIterator(filepath.Join(homedir, ".local/share/fish/fish_history"))
|
||||
lines := readFileToIterator(getFishHistoryPath(homedir))
|
||||
return func(yield func(string, error) bool) bool {
|
||||
return lines(func(line string, err error) bool {
|
||||
if err != nil {
|
||||
@ -646,6 +722,35 @@ func EncryptAndMarshal(config *hctx.ClientConfig, entries []*data.HistoryEntry)
|
||||
return jsonValue, nil
|
||||
}
|
||||
|
||||
func forEach[T any](arr []T, numThreads int, fn func(T) error) error {
|
||||
wg := &sync.WaitGroup{}
|
||||
wg.Add(len(arr))
|
||||
|
||||
limiter := make(chan bool, numThreads)
|
||||
|
||||
var errors []error
|
||||
for _, item := range arr {
|
||||
limiter <- true
|
||||
go func(x T) {
|
||||
defer wg.Done()
|
||||
err := fn(x)
|
||||
if err != nil {
|
||||
errors = append(errors, err)
|
||||
}
|
||||
<-limiter
|
||||
}(item)
|
||||
if len(errors) > 0 {
|
||||
return errors[0]
|
||||
}
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
if len(errors) > 0 {
|
||||
return errors[0]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Reupload(ctx context.Context) error {
|
||||
config := hctx.GetConf(ctx)
|
||||
if config.IsOffline {
|
||||
@ -655,7 +760,15 @@ func Reupload(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to reupload due to failed search: %w", err)
|
||||
}
|
||||
for _, chunk := range shared.Chunks(entries, 100) {
|
||||
var bar *progressbar.ProgressBar
|
||||
if len(entries) > NUM_IMPORTED_ENTRIES_SLOW {
|
||||
fmt.Println("Persisting history entries")
|
||||
bar = progressbar.Default(int64(len(entries)))
|
||||
defer bar.Finish()
|
||||
}
|
||||
chunkSize := 500
|
||||
chunks := shared.Chunks(entries, chunkSize)
|
||||
return forEach(chunks, 10, func(chunk []*data.HistoryEntry) error {
|
||||
jsonValue, err := EncryptAndMarshal(config, chunk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to reupload due to failed encryption: %w", err)
|
||||
@ -664,8 +777,11 @@ func Reupload(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to reupload due to failed POST: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
if bar != nil {
|
||||
_ = bar.Add(chunkSize)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func RetrieveAdditionalEntriesFromRemote(ctx context.Context) error {
|
||||
|
||||
8
go.mod
8
go.mod
@ -15,7 +15,7 @@ require (
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/jackc/pgx/v4 v4.14.1
|
||||
github.com/lib/pq v1.10.4
|
||||
github.com/mattn/go-runewidth v0.0.14
|
||||
github.com/mattn/go-runewidth v0.0.15
|
||||
github.com/muesli/termenv v0.13.0
|
||||
github.com/rodaine/table v1.0.1
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
@ -179,11 +179,12 @@ require (
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.15 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/miekg/pkcs11 v1.1.1 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
@ -207,9 +208,10 @@ require (
|
||||
github.com/prometheus/common v0.37.0 // indirect
|
||||
github.com/prometheus/procfs v0.8.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/rivo/uniseg v0.4.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.4 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/sassoftware/relic v0.0.0-20210427151427-dfb082b79b74 // indirect
|
||||
github.com/schollz/progressbar/v3 v3.13.1 // indirect
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect
|
||||
github.com/segmentio/ksuid v1.0.4 // indirect
|
||||
github.com/shibumi/go-pathspec v1.3.0 // indirect
|
||||
|
||||
11
go.sum
11
go.sum
@ -1008,6 +1008,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
|
||||
github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
|
||||
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
|
||||
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
|
||||
@ -1087,6 +1088,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
@ -1097,6 +1100,8 @@ github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRC
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
|
||||
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
@ -1114,6 +1119,8 @@ github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WT
|
||||
github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
|
||||
github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
|
||||
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
|
||||
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
|
||||
@ -1315,6 +1322,8 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=
|
||||
github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
|
||||
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
@ -1344,6 +1353,8 @@ github.com/sassoftware/go-rpmutils v0.1.1/go.mod h1:euhXULoBpvAxqrBHEyJS4Tsu3hHx
|
||||
github.com/sassoftware/relic v0.0.0-20210427151427-dfb082b79b74 h1:sUNzanSKA9z/h8xXl+ZJoxIYZL0Qx306MmxqRrvUgr0=
|
||||
github.com/sassoftware/relic v0.0.0-20210427151427-dfb082b79b74/go.mod h1:YlB8wFIZmFLZ1JllNBfSURzz52fBxbliNgYALk1UDmk=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
|
||||
github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
|
||||
github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/FulN9fTtqYUdS5+Oxzt+DUE=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user