From 9da18eb7d55dbaabd52a2aa294206e4bf6e7ba40 Mon Sep 17 00:00:00 2001
From: David Dworken <david@daviddworken.com>
Date: Mon, 10 Oct 2022 22:04:59 -0700
Subject: [PATCH] Chunk the uploads when reuploading to avoid having one giant
 request

---
 client/lib/lib.go      | 28 +++++++++++++++++++++-------
 client/lib/lib_test.go | 20 ++++++++++++++++++++
 2 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/client/lib/lib.go b/client/lib/lib.go
index 00e16d8..5348d05 100644
--- a/client/lib/lib.go
+++ b/client/lib/lib.go
@@ -1089,13 +1089,27 @@ func Reupload(ctx *context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to reupload due to failed search: %v", err)
 	}
-	jsonValue, err := EncryptAndMarshal(config, entries)
-	if err != nil {
-		return fmt.Errorf("failed to reupload due to failed encryption: %v", err)
-	}
-	_, err = ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
-	if err != nil {
-		return fmt.Errorf("failed to reupload due to failed POST: %v", err)
+	for _, chunk := range chunks(entries, 100) {
+		jsonValue, err := EncryptAndMarshal(config, chunk)
+		if err != nil {
+			return fmt.Errorf("failed to reupload due to failed encryption: %v", err)
+		}
+		_, err = ApiPost("/api/v1/submit?source_device_id="+config.DeviceId, "application/json", jsonValue)
+		if err != nil {
+			return fmt.Errorf("failed to reupload due to failed POST: %v", err)
+		}
 	}
 	return nil
 }
+
+func chunks[k any](slice []k, chunkSize int) [][]k {
+	var chunks [][]k
+	for i := 0; i < len(slice); i += chunkSize {
+		end := i + chunkSize
+		if end > len(slice) {
+			end = len(slice)
+		}
+		chunks = append(chunks, slice[i:end])
+	}
+	return chunks
+}
diff --git a/client/lib/lib_test.go b/client/lib/lib_test.go
index 5cd3fcd..02e1d46 100644
--- a/client/lib/lib_test.go
+++ b/client/lib/lib_test.go
@@ -4,6 +4,7 @@ import (
 	"os"
 	"os/user"
 	"path"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -283,3 +284,22 @@ func TestMaybeSkipBashHistTimePrefix(t *testing.T) {
 		}
 	}
 }
+
+func TestChunks(t *testing.T) {
+	testcases := []struct {
+		input     []int
+		chunkSize int
+		output    [][]int
+	}{
+		{[]int{1, 2, 3, 4, 5}, 2, [][]int{{1, 2}, {3, 4}, {5}}},
+		{[]int{1, 2, 3, 4, 5}, 3, [][]int{{1, 2, 3}, {4, 5}}},
+		{[]int{1, 2, 3, 4, 5}, 1, [][]int{{1}, {2}, {3}, {4}, {5}}},
+		{[]int{1, 2, 3, 4, 5}, 4, [][]int{{1, 2, 3, 4}, {5}}},
+	}
+	for _, tc := range testcases {
+		actual := chunks(tc.input, tc.chunkSize)
+		if !reflect.DeepEqual(actual, tc.output) {
+			t.Fatal("chunks failure")
+		}
+	}
+}