vfs: add tests for downloaders

vfs/vfscache/downloaders/downloaders_test.go

@@ -0,0 +1,132 @@
+package downloaders
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"sync"
+	"testing"
+	"time"
+
+	_ "github.com/rclone/rclone/backend/local"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/lib/ranges"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/vfs/vfscommon"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestMain drives the tests
+func TestMain(m *testing.M) {
+	fstest.TestMain(m)
+}
+
+type testItem struct {
+	mu   sync.Mutex
+	t    *testing.T
+	rs   ranges.Ranges
+	size int64
+}
+
+// HasRange returns true if the current ranges entirely include range
+func (item *testItem) HasRange(r ranges.Range) bool {
+	item.mu.Lock()
+	defer item.mu.Unlock()
+	return item.rs.Present(r)
+}
+
+// FindMissing adjusts r returning a new ranges.Range which only
+// contains the range which needs to be downloaded. This could be
+// empty - check with IsEmpty. It also adjust this to make sure it is
+// not larger than the file.
+func (item *testItem) FindMissing(r ranges.Range) (outr ranges.Range) {
+	item.mu.Lock()
+	defer item.mu.Unlock()
+	outr = item.rs.FindMissing(r)
+	// Clip returned block to size of file
+	outr.Clip(item.size)
+	return outr
+}
+
+// WriteAtNoOverwrite writes b to the file, but will not overwrite
+// already present ranges.
+//
+// This is used by the downloader to write bytes to the file
+//
+// It returns n the total bytes processed and skipped the number of
+// bytes which were processed but not actually written to the file.
+func (item *testItem) WriteAtNoOverwrite(b []byte, off int64) (n int, skipped int, err error) {
+	item.mu.Lock()
+	defer item.mu.Unlock()
+	item.rs.Insert(ranges.Range{Pos: off, Size: int64(len(b))})
+
+	// Check contents is correct
+	in := readers.NewPatternReader(item.size)
+	checkBuf := make([]byte, len(b))
+	_, err = in.Seek(off, io.SeekStart)
+	require.NoError(item.t, err)
+	n, _ = in.Read(checkBuf)
+	require.Equal(item.t, len(b), n)
+	assert.Equal(item.t, checkBuf, b)
+
+	return n, 0, nil
+}
+
+func TestDownloaders(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	var (
+		ctx    = context.Background()
+		remote = "potato.txt"
+		size   = int64(50*1024*1024 - 1234)
+	)
+
+	// Write the test file
+	in := ioutil.NopCloser(readers.NewPatternReader(size))
+	src, err := operations.RcatSize(ctx, r.Fremote, remote, in, size, time.Now())
+	require.NoError(t, err)
+	assert.Equal(t, size, src.Size())
+
+	newTest := func() (*testItem, *Downloaders) {
+		item := &testItem{
+			t:    t,
+			size: size,
+		}
+		opt := vfscommon.DefaultOpt
+		dls := New(item, &opt, remote, src)
+		return item, dls
+	}
+	cancel := func(dls *Downloaders) {
+		assert.NoError(t, dls.Close(nil))
+	}
+
+	t.Run("Download", func(t *testing.T) {
+		item, dls := newTest()
+		defer cancel(dls)
+
+		for _, r := range []ranges.Range{
+			{Pos: 100, Size: 250},
+			{Pos: 500, Size: 250},
+			{Pos: 25000000, Size: 250},
+		} {
+			err := dls.Download(r)
+			require.NoError(t, err)
+			assert.True(t, item.HasRange(r))
+		}
+	})
+
+	t.Run("EnsureDownloader", func(t *testing.T) {
+		item, dls := newTest()
+		defer cancel(dls)
+		r := ranges.Range{Pos: 40 * 1024 * 1024, Size: 250}
+		err := dls.EnsureDownloader(r)
+		require.NoError(t, err)
+		// FIXME racy test
+		assert.False(t, item.HasRange(r))
+		time.Sleep(time.Second)
+		assert.True(t, item.HasRange(r))
+	})
+}