extern/hishtory
1
0
Fork 0
mirror of https://github.com/ddworken/hishtory.git synced 2025-03-28 16:46:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

Chunk the uploads when reuploading to avoid having one giant request

Browse Source
This commit is contained in:
David Dworken 2022-10-10 22:04:59 -07:00
parent 8e6a55237c
commit 9da18eb7d5
2 changed files with 41 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
client/lib/lib.go
View File

@ -1089,13 +1089,27 @@ func Reupload(ctx *context.Context) error {
if err != nil {
return fmt.Errorf("failed to reupload due to failed search: %v", err)
}
jsonValue, err := EncryptAndMarshal(config, entries)
if err != nil {
return fmt.Errorf("failed to reupload due to failed encryption: %v", err)
}
_, err = ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
if err != nil {
return fmt.Errorf("failed to reupload due to failed POST: %v", err)
for _, chunk := range chunks(entries, 100) {
jsonValue, err := EncryptAndMarshal(config, chunk)
if err != nil {
return fmt.Errorf("failed to reupload due to failed encryption: %v", err)
}
_, err = ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
if err != nil {
return fmt.Errorf("failed to reupload due to failed POST: %v", err)
}
}
return nil
}
func chunks[k any](slice []k, chunkSize int) [][]k {
var chunks [][]k
for i := 0; i < len(slice); i += chunkSize {
end := i + chunkSize
if end > len(slice) {
end = len(slice)
}
chunks = append(chunks, slice[i:end])
}
return chunks
}

20
client/lib/lib_test.go
View File

@ -4,6 +4,7 @@ import (
"os"
"os/user"
"path"
"reflect"
"strings"
"testing"
"time"
@ -283,3 +284,22 @@ func TestMaybeSkipBashHistTimePrefix(t *testing.T) {
}
}
}
func TestChunks(t *testing.T) {
testcases := []struct {
input []int
chunkSize int
output [][]int
}{
{[]int{1, 2, 3, 4, 5}, 2, [][]int{{1, 2}, {3, 4}, {5}}},
{[]int{1, 2, 3, 4, 5}, 3, [][]int{{1, 2, 3}, {4, 5}}},
{[]int{1, 2, 3, 4, 5}, 1, [][]int{{1}, {2}, {3}, {4}, {5}}},
{[]int{1, 2, 3, 4, 5}, 4, [][]int{{1, 2, 3, 4}, {5}}},
}
for _, tc := range testcases {
actual := chunks(tc.input, tc.chunkSize)
if !reflect.DeepEqual(actual, tc.output) {
t.Fatal("chunks failure")
}
}
}