extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-08-15 16:22:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

chunkedreader: add --vfs-read-chunk-streams to parallel read chunks

Browse Source 
This converts the ChunkedReader into an interface and provides two
implementations one sequential and one parallel.

This can be used to improve the performance of the VFS on high
bandwidth or high latency links.

Fixes #4760
This commit is contained in:
Nick Craig-Wood
2024-03-12 16:57:16 +00:00
parent 10270a4354
commit 27b281ef69
11 changed files with 835 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
fs/chunkedreader/chunkedreader_test.go
View File

@ -7,21 +7,47 @@ import (
"math/rand"
"testing"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/mockobject"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestChunkedReader(t *testing.T) {
content := makeContent(t, 1024)
// TestMain drives the tests
func TestMain(m *testing.M) {
fstest.TestMain(m)
}
for _, mode := range mockobject.SeekModes {
t.Run(mode.String(), testRead(content, mode))
func TestChunkedReader(t *testing.T) {
ctx := context.Background()
o := mockobject.New("test.bin").WithContent([]byte("hello"), mockobject.SeekModeRegular)
const MB = 1024 * 1024
for _, test := range []struct {
initialChunkSize int64
maxChunkSize int64
streams int
crType any
unknownSize bool
}{
{-1, MB, 0, new(sequential), false},
{MB, 10 * MB, 0, new(sequential), false},
{MB, 10 * MB, 1, new(sequential), false},
{MB, 10 * MB, 1, new(sequential), true},
{MB, 10 * MB, 2, new(parallel), false},
{MB, 10 * MB, 2, new(sequential), true},
} {
what := fmt.Sprintf("%+v", test)
o.SetUnknownSize(test.unknownSize)
cr := New(ctx, o, test.initialChunkSize, test.maxChunkSize, test.streams)
assert.IsType(t, test.crType, cr, what)
require.NoError(t, cr.Close(), what)
}
}
func testRead(content []byte, mode mockobject.SeekMode) func(*testing.T) {
func testRead(content []byte, mode mockobject.SeekMode, streams int) func(*testing.T) {
return func(t *testing.T) {
ctx := context.Background()
chunkSizes := []int64{-1, 0, 1, 15, 16, 17, 1023, 1024, 1025, 2000}
offsets := []int64{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33,
63, 64, 65, 511, 512, 513, 1023, 1024, 1025}
@ -39,13 +65,13 @@ func testRead(content []byte, mode mockobject.SeekMode) func(*testing.T) {
}
t.Run(fmt.Sprintf("Chunksize_%d_%d", cs, csMax), func(t *testing.T) {
cr := New(context.Background(), o, cs, csMax)
cr := New(ctx, o, cs, csMax, streams)
for _, offset := range offsets {
for _, limit := range limits {
what := fmt.Sprintf("offset %d, limit %d", offset, limit)
p, err := cr.RangeSeek(context.Background(), offset, io.SeekStart, limit)
p, err := cr.RangeSeek(ctx, offset, io.SeekStart, limit)
if offset >= cl {
require.Error(t, err, what)
return
@ -74,32 +100,33 @@ func testRead(content []byte, mode mockobject.SeekMode) func(*testing.T) {
}
}
func TestErrorAfterClose(t *testing.T) {
func testErrorAfterClose(t *testing.T, streams int) {
ctx := context.Background()
content := makeContent(t, 1024)
o := mockobject.New("test.bin").WithContent(content, mockobject.SeekModeNone)
// Close
cr := New(context.Background(), o, 0, 0)
cr := New(ctx, o, 0, 0, streams)
require.NoError(t, cr.Close())
require.Error(t, cr.Close())
// Read
cr = New(context.Background(), o, 0, 0)
cr = New(ctx, o, 0, 0, streams)
require.NoError(t, cr.Close())
var buf [1]byte
_, err := cr.Read(buf[:])
require.Error(t, err)
// Seek
cr = New(context.Background(), o, 0, 0)
cr = New(ctx, o, 0, 0, streams)
require.NoError(t, cr.Close())
_, err = cr.Seek(1, io.SeekCurrent)
require.Error(t, err)
// RangeSeek
cr = New(context.Background(), o, 0, 0)
cr = New(ctx, o, 0, 0, streams)
require.NoError(t, cr.Close())
_, err = cr.RangeSeek(context.Background(), 1, io.SeekCurrent, 0)
_, err = cr.RangeSeek(ctx, 1, io.SeekCurrent, 0)
require.Error(t, err)
}