rclone/vfs/vfscache/downloaders/downloaders_test.go

package downloaders

import (
	"context"
	"io"
	"io/ioutil"
	"sync"
	"testing"
	"time"

	_ "github.com/rclone/rclone/backend/local"
	"github.com/rclone/rclone/fs/operations"
	"github.com/rclone/rclone/fstest"
	"github.com/rclone/rclone/lib/ranges"
	"github.com/rclone/rclone/lib/readers"
	"github.com/rclone/rclone/vfs/vfscommon"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// TestMain drives the tests
func TestMain(m *testing.M) {
	fstest.TestMain(m)
}

type testItem struct {
	mu   sync.Mutex
	t    *testing.T
	rs   ranges.Ranges
	size int64
}

// HasRange returns true if the current ranges entirely include range
func (item *testItem) HasRange(r ranges.Range) bool {
	item.mu.Lock()
	defer item.mu.Unlock()
	return item.rs.Present(r)
}

// FindMissing adjusts r returning a new ranges.Range which only
// contains the range which needs to be downloaded. This could be
// empty - check with IsEmpty. It also adjust this to make sure it is
// not larger than the file.
func (item *testItem) FindMissing(r ranges.Range) (outr ranges.Range) {
	item.mu.Lock()
	defer item.mu.Unlock()
	outr = item.rs.FindMissing(r)
	// Clip returned block to size of file
	outr.Clip(item.size)
	return outr
}

// WriteAtNoOverwrite writes b to the file, but will not overwrite
// already present ranges.
//
// This is used by the downloader to write bytes to the file
//
// It returns n the total bytes processed and skipped the number of
// bytes which were processed but not actually written to the file.
func (item *testItem) WriteAtNoOverwrite(b []byte, off int64) (n int, skipped int, err error) {
	item.mu.Lock()
	defer item.mu.Unlock()
	item.rs.Insert(ranges.Range{Pos: off, Size: int64(len(b))})

	// Check contents is correct
	in := readers.NewPatternReader(item.size)
	checkBuf := make([]byte, len(b))
	_, err = in.Seek(off, io.SeekStart)
	require.NoError(item.t, err)
	n, _ = in.Read(checkBuf)
	require.Equal(item.t, len(b), n)
	assert.Equal(item.t, checkBuf, b)

	return n, 0, nil
}

func TestDownloaders(t *testing.T) {
	r := fstest.NewRun(t)
	defer r.Finalise()

	var (
		ctx    = context.Background()
		remote = "potato.txt"
		size   = int64(50*1024*1024 - 1234)
	)

	// Write the test file
	in := ioutil.NopCloser(readers.NewPatternReader(size))
	src, err := operations.RcatSize(ctx, r.Fremote, remote, in, size, time.Now())
	require.NoError(t, err)
	assert.Equal(t, size, src.Size())

	newTest := func() (*testItem, *Downloaders) {
		item := &testItem{
			t:    t,
			size: size,
		}
		opt := vfscommon.DefaultOpt
		dls := New(item, &opt, remote, src)
		return item, dls
	}
	cancel := func(dls *Downloaders) {
		assert.NoError(t, dls.Close(nil))
	}

	t.Run("Download", func(t *testing.T) {
		item, dls := newTest()
		defer cancel(dls)

		for _, r := range []ranges.Range{
			{Pos: 100, Size: 250},
			{Pos: 500, Size: 250},
			{Pos: 25000000, Size: 250},
		} {
			err := dls.Download(r)
			require.NoError(t, err)
			assert.True(t, item.HasRange(r))
		}
	})

	t.Run("EnsureDownloader", func(t *testing.T) {
		item, dls := newTest()
		defer cancel(dls)
		r := ranges.Range{Pos: 40 * 1024 * 1024, Size: 250}
		err := dls.EnsureDownloader(r)
		require.NoError(t, err)
		// FIXME racy test
		assert.False(t, item.HasRange(r))
		time.Sleep(time.Second)
		assert.True(t, item.HasRange(r))
	})
}