mirror of
https://github.com/rclone/rclone.git
synced 2024-12-27 01:19:12 +01:00
2701 lines
89 KiB
Go
2701 lines
89 KiB
Go
// Package fstests provides generic integration tests for the Fs and
|
|
// Object interfaces.
|
|
//
|
|
// These tests are concerned with the basic functionality of a
|
|
// backend. The tests in fs/sync and fs/operations tests more
|
|
// cornercases that these tests don't.
|
|
package fstests
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"math/bits"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"reflect"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/cache"
|
|
"github.com/rclone/rclone/fs/config"
|
|
"github.com/rclone/rclone/fs/fserrors"
|
|
"github.com/rclone/rclone/fs/fspath"
|
|
"github.com/rclone/rclone/fs/hash"
|
|
"github.com/rclone/rclone/fs/object"
|
|
"github.com/rclone/rclone/fs/operations"
|
|
"github.com/rclone/rclone/fs/walk"
|
|
"github.com/rclone/rclone/fstest"
|
|
"github.com/rclone/rclone/fstest/testserver"
|
|
"github.com/rclone/rclone/lib/encoder"
|
|
"github.com/rclone/rclone/lib/random"
|
|
"github.com/rclone/rclone/lib/readers"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// InternalTester is an optional interface for Fs which allows to execute internal tests
|
|
//
|
|
// This interface should be implemented in 'backend'_internal_test.go and not in 'backend'.go
|
|
type InternalTester interface {
|
|
InternalTest(*testing.T)
|
|
}
|
|
|
|
// ChunkedUploadConfig contains the values used by TestFsPutChunked
|
|
// to determine the limits of chunked uploading
|
|
type ChunkedUploadConfig struct {
|
|
// Minimum allowed chunk size
|
|
MinChunkSize fs.SizeSuffix
|
|
// Maximum allowed chunk size, 0 is no limit
|
|
MaxChunkSize fs.SizeSuffix
|
|
// Rounds the given chunk size up to the next valid value
|
|
// nil will disable rounding
|
|
// e.g. the next power of 2
|
|
CeilChunkSize func(fs.SizeSuffix) fs.SizeSuffix
|
|
// More than one chunk is required on upload
|
|
NeedMultipleChunks bool
|
|
// Skip this particular remote
|
|
Skip bool
|
|
}
|
|
|
|
// SetUploadChunkSizer is a test only interface to change the upload chunk size at runtime
|
|
type SetUploadChunkSizer interface {
|
|
// Change the configured UploadChunkSize.
|
|
// Will only be called while no transfer is in progress.
|
|
SetUploadChunkSize(fs.SizeSuffix) (fs.SizeSuffix, error)
|
|
}
|
|
|
|
// SetUploadCutoffer is a test only interface to change the upload cutoff size at runtime
|
|
type SetUploadCutoffer interface {
|
|
// Change the configured UploadCutoff.
|
|
// Will only be called while no transfer is in progress.
|
|
SetUploadCutoff(fs.SizeSuffix) (fs.SizeSuffix, error)
|
|
}
|
|
|
|
// SetCopyCutoffer is a test only interface to change the copy cutoff size at runtime
|
|
type SetCopyCutoffer interface {
|
|
// Change the configured CopyCutoff.
|
|
// Will only be called while no transfer is in progress.
|
|
SetCopyCutoff(fs.SizeSuffix) (fs.SizeSuffix, error)
|
|
}
|
|
|
|
// NextPowerOfTwo returns the current or next bigger power of two.
|
|
// All values less or equal 0 will return 0
|
|
func NextPowerOfTwo(i fs.SizeSuffix) fs.SizeSuffix {
|
|
return 1 << uint(64-bits.LeadingZeros64(uint64(i)-1))
|
|
}
|
|
|
|
// NextMultipleOf returns a function that can be used as a CeilChunkSize function.
|
|
// This function will return the next multiple of m that is equal or bigger than i.
|
|
// All values less or equal 0 will return 0.
|
|
func NextMultipleOf(m fs.SizeSuffix) func(fs.SizeSuffix) fs.SizeSuffix {
|
|
if m <= 0 {
|
|
panic(fmt.Sprintf("invalid multiplier %s", m))
|
|
}
|
|
return func(i fs.SizeSuffix) fs.SizeSuffix {
|
|
if i <= 0 {
|
|
return 0
|
|
}
|
|
|
|
return (((i - 1) / m) + 1) * m
|
|
}
|
|
}
|
|
|
|
// dirsToNames returns a sorted list of names
|
|
func dirsToNames(dirs []fs.Directory) []string {
|
|
names := []string{}
|
|
for _, dir := range dirs {
|
|
names = append(names, fstest.Normalize(dir.Remote()))
|
|
}
|
|
sort.Strings(names)
|
|
return names
|
|
}
|
|
|
|
// objsToNames returns a sorted list of object names
|
|
func objsToNames(objs []fs.Object) []string {
|
|
names := []string{}
|
|
for _, obj := range objs {
|
|
names = append(names, fstest.Normalize(obj.Remote()))
|
|
}
|
|
sort.Strings(names)
|
|
return names
|
|
}
|
|
|
|
// retry f() until no retriable error
|
|
func retry(t *testing.T, what string, f func() error) {
|
|
const maxTries = 10
|
|
var err error
|
|
for tries := 1; tries <= maxTries; tries++ {
|
|
err = f()
|
|
// exit if no error, or error is not retriable
|
|
if err == nil || !fserrors.IsRetryError(err) {
|
|
break
|
|
}
|
|
t.Logf("%s error: %v - low level retry %d/%d", what, err, tries, maxTries)
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
require.NoError(t, err, what)
|
|
}
|
|
|
|
// check interface
|
|
|
|
// PutTestContentsMetadata puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove
|
|
//
|
|
// It uploads the object with the mimeType and metadata passed in if set.
|
|
//
|
|
// It returns the object which will have been checked if check is set
|
|
func PutTestContentsMetadata(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool, mimeType string, metadata fs.Metadata) fs.Object {
|
|
var (
|
|
err error
|
|
obj fs.Object
|
|
uploadHash *hash.MultiHasher
|
|
)
|
|
retry(t, "Put", func() error {
|
|
buf := bytes.NewBufferString(contents)
|
|
uploadHash = hash.NewMultiHasher()
|
|
in := io.TeeReader(buf, uploadHash)
|
|
|
|
file.Size = int64(buf.Len())
|
|
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
|
|
if mimeType != "" || metadata != nil {
|
|
// force the --metadata flag on temporarily
|
|
if metadata != nil {
|
|
ci := fs.GetConfig(ctx)
|
|
previousMetadata := ci.Metadata
|
|
ci.Metadata = true
|
|
defer func() {
|
|
ci.Metadata = previousMetadata
|
|
}()
|
|
}
|
|
obji.WithMetadata(metadata).WithMimeType(mimeType)
|
|
}
|
|
obj, err = f.Put(ctx, in, obji)
|
|
return err
|
|
})
|
|
file.Hashes = uploadHash.Sums()
|
|
if check {
|
|
// Overwrite time with that in metadata if it is already specified
|
|
mtime, ok := metadata["mtime"]
|
|
if ok {
|
|
modTime, err := time.Parse(time.RFC3339Nano, mtime)
|
|
require.NoError(t, err)
|
|
file.ModTime = modTime
|
|
}
|
|
file.Check(t, obj, f.Precision())
|
|
// Re-read the object and check again
|
|
obj = fstest.NewObject(ctx, t, f, file.Path)
|
|
file.Check(t, obj, f.Precision())
|
|
}
|
|
return obj
|
|
}
|
|
|
|
// PutTestContents puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove
|
|
func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool) fs.Object {
|
|
return PutTestContentsMetadata(ctx, t, f, file, contents, check, "", nil)
|
|
}
|
|
|
|
// testPut puts file with random contents to the remote
|
|
func testPut(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) {
|
|
contents := random.String(100)
|
|
return contents, PutTestContents(ctx, t, f, file, contents, true)
|
|
}
|
|
|
|
// testPutMimeType puts file with random contents to the remote and the mime type given
|
|
func testPutMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, mimeType string, metadata fs.Metadata) (string, fs.Object) {
|
|
contents := random.String(100)
|
|
return contents, PutTestContentsMetadata(ctx, t, f, file, contents, true, mimeType, metadata)
|
|
}
|
|
|
|
// testPutLarge puts file to the remote, checks it and removes it on success.
|
|
//
|
|
// If stream is set, then it uploads the file with size -1
|
|
func testPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, stream bool) {
|
|
var (
|
|
err error
|
|
obj fs.Object
|
|
uploadHash *hash.MultiHasher
|
|
)
|
|
retry(t, "PutLarge", func() error {
|
|
r := readers.NewPatternReader(file.Size)
|
|
uploadHash = hash.NewMultiHasher()
|
|
in := io.TeeReader(r, uploadHash)
|
|
|
|
size := file.Size
|
|
if stream {
|
|
size = -1
|
|
}
|
|
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, size, true, nil, nil)
|
|
obj, err = f.Put(ctx, in, obji)
|
|
if file.Size == 0 && err == fs.ErrorCantUploadEmptyFiles {
|
|
t.Skip("Can't upload zero length files")
|
|
}
|
|
return err
|
|
})
|
|
file.Hashes = uploadHash.Sums()
|
|
file.Check(t, obj, f.Precision())
|
|
|
|
// Re-read the object and check again
|
|
obj = fstest.NewObject(ctx, t, f, file.Path)
|
|
file.Check(t, obj, f.Precision())
|
|
|
|
// Download the object and check it is OK
|
|
downloadHash := hash.NewMultiHasher()
|
|
download, err := obj.Open(ctx)
|
|
require.NoError(t, err)
|
|
n, err := io.Copy(downloadHash, download)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, file.Size, n)
|
|
require.NoError(t, download.Close())
|
|
assert.Equal(t, file.Hashes, downloadHash.Sums())
|
|
|
|
// Remove the object
|
|
require.NoError(t, obj.Remove(ctx))
|
|
}
|
|
|
|
// TestPutLarge puts file to the remote, checks it and removes it on success.
|
|
func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) {
|
|
testPutLarge(ctx, t, f, file, false)
|
|
}
|
|
|
|
// TestPutLargeStreamed puts file of unknown size to the remote, checks it and removes it on success.
|
|
func TestPutLargeStreamed(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) {
|
|
testPutLarge(ctx, t, f, file, true)
|
|
}
|
|
|
|
// ReadObject reads the contents of an object as a string
|
|
func ReadObject(ctx context.Context, t *testing.T, obj fs.Object, limit int64, options ...fs.OpenOption) string {
|
|
what := fmt.Sprintf("readObject(%q) limit=%d, options=%+v", obj, limit, options)
|
|
in, err := obj.Open(ctx, options...)
|
|
require.NoError(t, err, what)
|
|
var r io.Reader = in
|
|
if limit >= 0 {
|
|
r = &io.LimitedReader{R: r, N: limit}
|
|
}
|
|
contents, err := io.ReadAll(r)
|
|
require.NoError(t, err, what)
|
|
err = in.Close()
|
|
require.NoError(t, err, what)
|
|
return string(contents)
|
|
}
|
|
|
|
// ExtraConfigItem describes a config item for the tests
|
|
type ExtraConfigItem struct{ Name, Key, Value string }
|
|
|
|
// Opt is options for Run
|
|
type Opt struct {
|
|
RemoteName string
|
|
NilObject fs.Object
|
|
ExtraConfig []ExtraConfigItem
|
|
SkipBadWindowsCharacters bool // skips unusable characters for windows if set
|
|
SkipFsMatch bool // if set skip exact matching of Fs value
|
|
TiersToTest []string // List of tiers which can be tested in setTier test
|
|
ChunkedUpload ChunkedUploadConfig
|
|
UnimplementableFsMethods []string // List of Fs methods which can't be implemented in this wrapping Fs
|
|
UnimplementableObjectMethods []string // List of Object methods which can't be implemented in this wrapping Fs
|
|
UnimplementableDirectoryMethods []string // List of Directory methods which can't be implemented in this wrapping Fs
|
|
SkipFsCheckWrap bool // if set skip FsCheckWrap
|
|
SkipObjectCheckWrap bool // if set skip ObjectCheckWrap
|
|
SkipDirectoryCheckWrap bool // if set skip DirectoryCheckWrap
|
|
SkipInvalidUTF8 bool // if set skip invalid UTF-8 checks
|
|
SkipLeadingDot bool // if set skip leading dot checks
|
|
QuickTestOK bool // if set, run this test with make quicktest
|
|
}
|
|
|
|
// returns true if x is found in ss
|
|
func stringsContains(x string, ss []string) bool {
|
|
for _, s := range ss {
|
|
if x == s {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// toUpperASCII returns a copy of the string s with all Unicode
|
|
// letters mapped to their upper case.
|
|
func toUpperASCII(s string) string {
|
|
return strings.Map(func(r rune) rune {
|
|
if 'a' <= r && r <= 'z' {
|
|
r -= 'a' - 'A'
|
|
}
|
|
return r
|
|
}, s)
|
|
}
|
|
|
|
// removeConfigID removes any {xyz} parts of the name put in for
|
|
// config disambiguation
|
|
func removeConfigID(s string) string {
|
|
bra := strings.IndexRune(s, '{')
|
|
ket := strings.IndexRune(s, '}')
|
|
if bra >= 0 && ket > bra {
|
|
s = s[:bra] + s[ket+1:]
|
|
}
|
|
return s
|
|
}
|
|
|
|
// InternalTestFiles is the state of the remote at the moment the internal tests are called
|
|
var InternalTestFiles []fstest.Item
|
|
|
|
// Run runs the basic integration tests for a remote using the options passed in.
|
|
//
|
|
// They are structured in a hierarchical way so that dependencies for the tests can be created.
|
|
//
|
|
// For example some tests require the directory to be created - these
|
|
// are inside the "FsMkdir" test. Some tests require some tests files
|
|
// - these are inside the "FsPutFiles" test.
|
|
func Run(t *testing.T, opt *Opt) {
|
|
var (
|
|
f fs.Fs
|
|
remoteName = opt.RemoteName
|
|
subRemoteName string
|
|
subRemoteLeaf string
|
|
file1 = fstest.Item{
|
|
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
|
Path: "file name.txt",
|
|
}
|
|
file1Contents string
|
|
file1MimeType = "text/csv"
|
|
file1Metadata = fs.Metadata{"rclone-test": "potato"}
|
|
file2 = fstest.Item{
|
|
ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
|
|
Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`,
|
|
}
|
|
isLocalRemote bool
|
|
purged bool // whether the dir has been purged or not
|
|
ctx = context.Background()
|
|
ci = fs.GetConfig(ctx)
|
|
unwrappableFsMethods = []string{"Command"} // these Fs methods don't need to be wrapped ever
|
|
)
|
|
|
|
if strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" && !opt.QuickTestOK {
|
|
t.Skip("quicktest only")
|
|
}
|
|
|
|
// Skip the test if the remote isn't configured
|
|
skipIfNotOk := func(t *testing.T) {
|
|
if f == nil {
|
|
t.Skipf("WARN: %q not configured", remoteName)
|
|
}
|
|
}
|
|
|
|
// Skip if remote is not ListR capable, otherwise set the useListR
|
|
// flag, returning a function to restore its value
|
|
skipIfNotListR := func(t *testing.T) func() {
|
|
skipIfNotOk(t)
|
|
if f.Features().ListR == nil {
|
|
t.Skip("FS has no ListR interface")
|
|
}
|
|
previous := ci.UseListR
|
|
ci.UseListR = true
|
|
return func() {
|
|
ci.UseListR = previous
|
|
}
|
|
}
|
|
|
|
// Skip if remote is not SetTier and GetTier capable
|
|
skipIfNotSetTier := func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
if !f.Features().SetTier || !f.Features().GetTier {
|
|
t.Skip("FS has no SetTier & GetTier interfaces")
|
|
}
|
|
}
|
|
|
|
// Return true if f (or any of the things it wraps) is bucket
|
|
// based but not at the root.
|
|
isBucketBasedButNotRoot := func(f fs.Fs) bool {
|
|
f = fs.UnWrapFs(f)
|
|
return f.Features().BucketBased && strings.Contains(strings.Trim(f.Root(), "/"), "/")
|
|
}
|
|
|
|
// Initialise the remote
|
|
fstest.Initialise()
|
|
|
|
// Set extra config if supplied
|
|
for _, item := range opt.ExtraConfig {
|
|
config.FileSet(item.Name, item.Key, item.Value)
|
|
}
|
|
if *fstest.RemoteName != "" {
|
|
remoteName = *fstest.RemoteName
|
|
}
|
|
oldFstestRemoteName := fstest.RemoteName
|
|
fstest.RemoteName = &remoteName
|
|
defer func() {
|
|
fstest.RemoteName = oldFstestRemoteName
|
|
}()
|
|
t.Logf("Using remote %q", remoteName)
|
|
var err error
|
|
if remoteName == "" {
|
|
remoteName, err = fstest.LocalRemote()
|
|
require.NoError(t, err)
|
|
isLocalRemote = true
|
|
}
|
|
|
|
// Start any test servers if required
|
|
finish, err := testserver.Start(remoteName)
|
|
require.NoError(t, err)
|
|
defer finish()
|
|
|
|
// Make the Fs we are testing with, initialising the local variables
|
|
// subRemoteName - name of the remote after the TestRemote:
|
|
// subRemoteLeaf - a subdirectory to use under that
|
|
// remote - the result of fs.NewFs(TestRemote:subRemoteName)
|
|
subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(remoteName)
|
|
require.NoError(t, err)
|
|
f, err = fs.NewFs(context.Background(), subRemoteName)
|
|
if err == fs.ErrorNotFoundInConfigFile {
|
|
t.Logf("Didn't find %q in config file - skipping tests", remoteName)
|
|
return
|
|
}
|
|
require.NoError(t, err, fmt.Sprintf("unexpected error: %v", err))
|
|
|
|
// Get fsInfo which contains type, etc. of the fs
|
|
fsInfo, _, _, _, err := fs.ConfigFs(subRemoteName)
|
|
require.NoError(t, err, fmt.Sprintf("unexpected error: %v", err))
|
|
|
|
// Skip the rest if it failed
|
|
skipIfNotOk(t)
|
|
|
|
// Check to see if Fs that wrap other Fs implement all the optional methods
|
|
t.Run("FsCheckWrap", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
if opt.SkipFsCheckWrap {
|
|
t.Skip("Skipping FsCheckWrap on this Fs")
|
|
}
|
|
ft := new(fs.Features).Fill(ctx, f)
|
|
if ft.UnWrap == nil && !f.Features().Overlay {
|
|
t.Skip("Not a wrapping Fs")
|
|
}
|
|
v := reflect.ValueOf(ft).Elem()
|
|
vType := v.Type()
|
|
for i := 0; i < v.NumField(); i++ {
|
|
vName := vType.Field(i).Name
|
|
if stringsContains(vName, opt.UnimplementableFsMethods) {
|
|
continue
|
|
}
|
|
if stringsContains(vName, unwrappableFsMethods) {
|
|
continue
|
|
}
|
|
field := v.Field(i)
|
|
// skip the bools
|
|
if field.Type().Kind() == reflect.Bool {
|
|
continue
|
|
}
|
|
if field.IsNil() {
|
|
t.Errorf("Missing Fs wrapper for %s", vName)
|
|
}
|
|
}
|
|
})
|
|
|
|
// Check to see if Fs advertises commands and they work and have docs
|
|
t.Run("FsCommand", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
doCommand := f.Features().Command
|
|
if doCommand == nil {
|
|
t.Skip("No commands in this remote")
|
|
}
|
|
// Check the correct error is generated
|
|
_, err := doCommand(context.Background(), "NOTFOUND", nil, nil)
|
|
assert.Equal(t, fs.ErrorCommandNotFound, err, "Incorrect error generated on command not found")
|
|
// Check there are some commands in the fsInfo
|
|
fsInfo, _, _, _, err := fs.ConfigFs(remoteName)
|
|
require.NoError(t, err)
|
|
assert.True(t, len(fsInfo.CommandHelp) > 0, "Command is declared, must return some help in CommandHelp")
|
|
})
|
|
|
|
// TestFsRmdirNotFound tests deleting a nonexistent directory
|
|
t.Run("FsRmdirNotFound", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
if isBucketBasedButNotRoot(f) {
|
|
t.Skip("Skipping test as non root bucket-based remote")
|
|
}
|
|
err := f.Rmdir(ctx, "")
|
|
assert.Error(t, err, "Expecting error on Rmdir nonexistent")
|
|
})
|
|
|
|
// Make the directory
|
|
err = f.Mkdir(ctx, "")
|
|
require.NoError(t, err)
|
|
fstest.CheckListing(t, f, []fstest.Item{})
|
|
|
|
// TestFsString tests the String method
|
|
t.Run("FsString", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
str := f.String()
|
|
require.NotEqual(t, "", str)
|
|
})
|
|
|
|
// TestFsName tests the Name method
|
|
t.Run("FsName", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
got := removeConfigID(f.Name())
|
|
var want string
|
|
if isLocalRemote {
|
|
want = "local"
|
|
} else {
|
|
want = remoteName[:strings.LastIndex(remoteName, ":")]
|
|
comma := strings.IndexRune(remoteName, ',')
|
|
if comma >= 0 {
|
|
want = want[:comma]
|
|
}
|
|
}
|
|
require.Equal(t, want, got)
|
|
})
|
|
|
|
// TestFsRoot tests the Root method
|
|
t.Run("FsRoot", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
got := f.Root()
|
|
want := subRemoteName
|
|
colon := strings.LastIndex(want, ":")
|
|
if colon >= 0 {
|
|
want = want[colon+1:]
|
|
}
|
|
if isLocalRemote {
|
|
// only check last path element on local
|
|
require.Equal(t, filepath.Base(subRemoteName), filepath.Base(got))
|
|
} else {
|
|
require.Equal(t, want, got)
|
|
}
|
|
})
|
|
|
|
// TestFsRmdirEmpty tests deleting an empty directory
|
|
t.Run("FsRmdirEmpty", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
err := f.Rmdir(ctx, "")
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
// TestFsMkdir tests making a directory
|
|
//
|
|
// Tests that require the directory to be made are within this
|
|
t.Run("FsMkdir", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
err := f.Mkdir(ctx, "")
|
|
require.NoError(t, err)
|
|
fstest.CheckListing(t, f, []fstest.Item{})
|
|
|
|
err = f.Mkdir(ctx, "")
|
|
require.NoError(t, err)
|
|
|
|
// TestFsMkdirRmdirSubdir tests making and removing a sub directory
|
|
t.Run("FsMkdirRmdirSubdir", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
dir := "dir/subdir"
|
|
err := operations.Mkdir(ctx, f, dir)
|
|
require.NoError(t, err)
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{"dir", "dir/subdir"}, fs.GetModifyWindow(ctx, f))
|
|
|
|
err = operations.Rmdir(ctx, f, dir)
|
|
require.NoError(t, err)
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{"dir"}, fs.GetModifyWindow(ctx, f))
|
|
|
|
err = operations.Rmdir(ctx, f, "dir")
|
|
require.NoError(t, err)
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f))
|
|
})
|
|
|
|
// TestFsListEmpty tests listing an empty directory
|
|
t.Run("FsListEmpty", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
fstest.CheckListing(t, f, []fstest.Item{})
|
|
})
|
|
|
|
// TestFsListDirEmpty tests listing the directories from an empty directory
|
|
TestFsListDirEmpty := func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
objs, dirs, err := walk.GetAll(ctx, f, "", true, 1)
|
|
if !f.Features().CanHaveEmptyDirectories {
|
|
if err != fs.ErrorDirNotFound {
|
|
require.NoError(t, err)
|
|
}
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
assert.Equal(t, []string{}, objsToNames(objs))
|
|
assert.Equal(t, []string{}, dirsToNames(dirs))
|
|
}
|
|
t.Run("FsListDirEmpty", TestFsListDirEmpty)
|
|
|
|
// TestFsListRDirEmpty tests listing the directories from an empty directory using ListR
|
|
t.Run("FsListRDirEmpty", func(t *testing.T) {
|
|
defer skipIfNotListR(t)()
|
|
TestFsListDirEmpty(t)
|
|
})
|
|
|
|
// TestFsListDirNotFound tests listing the directories from an empty directory
|
|
TestFsListDirNotFound := func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
objs, dirs, err := walk.GetAll(ctx, f, "does not exist", true, 1)
|
|
if !f.Features().CanHaveEmptyDirectories {
|
|
if err != fs.ErrorDirNotFound {
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 0, len(objs)+len(dirs))
|
|
}
|
|
} else {
|
|
assert.Equal(t, fs.ErrorDirNotFound, err)
|
|
}
|
|
}
|
|
t.Run("FsListDirNotFound", TestFsListDirNotFound)
|
|
|
|
// TestFsListRDirNotFound tests listing the directories from an empty directory using ListR
|
|
t.Run("FsListRDirNotFound", func(t *testing.T) {
|
|
defer skipIfNotListR(t)()
|
|
TestFsListDirNotFound(t)
|
|
})
|
|
|
|
// FsEncoding tests that file name encodings are
|
|
// working by uploading a series of unusual files
|
|
// Must be run in an empty directory
|
|
t.Run("FsEncoding", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
if testing.Short() {
|
|
t.Skip("not running with -short")
|
|
}
|
|
|
|
// check no files or dirs as pre-requisite
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f))
|
|
|
|
for _, test := range []struct {
|
|
name string
|
|
path string
|
|
}{
|
|
// See lib/encoder/encoder.go for list of things that go here
|
|
{"control chars", "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\x7F"},
|
|
{"dot", "."},
|
|
{"dot dot", ".."},
|
|
{"punctuation", "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"},
|
|
{"leading space", " leading space"},
|
|
{"leading tilde", "~leading tilde"},
|
|
{"leading CR", "\rleading CR"},
|
|
{"leading LF", "\nleading LF"},
|
|
{"leading HT", "\tleading HT"},
|
|
{"leading VT", "\vleading VT"},
|
|
{"leading dot", ".leading dot"},
|
|
{"trailing space", "trailing space "},
|
|
{"trailing CR", "trailing CR\r"},
|
|
{"trailing LF", "trailing LF\n"},
|
|
{"trailing HT", "trailing HT\t"},
|
|
{"trailing VT", "trailing VT\v"},
|
|
{"trailing dot", "trailing dot."},
|
|
{"invalid UTF-8", "invalid utf-8\xfe"},
|
|
{"URL encoding", "test%46.txt"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
if opt.SkipInvalidUTF8 && test.name == "invalid UTF-8" {
|
|
t.Skip("Skipping " + test.name)
|
|
}
|
|
if opt.SkipLeadingDot && test.name == "leading dot" {
|
|
t.Skip("Skipping " + test.name)
|
|
}
|
|
// turn raw strings into Standard encoding
|
|
fileName := encoder.Standard.Encode(test.path)
|
|
dirName := fileName
|
|
t.Logf("testing %q", fileName)
|
|
assert.NoError(t, f.Mkdir(ctx, dirName))
|
|
file := fstest.Item{
|
|
ModTime: time.Now(),
|
|
Path: dirName + "/" + fileName, // test creating a file and dir with that name
|
|
}
|
|
_, o := testPut(context.Background(), t, f, &file)
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{file}, []string{dirName}, fs.GetModifyWindow(ctx, f))
|
|
assert.NoError(t, o.Remove(ctx))
|
|
assert.NoError(t, f.Rmdir(ctx, dirName))
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f))
|
|
})
|
|
}
|
|
})
|
|
|
|
// TestFsNewObjectNotFound tests not finding an object
|
|
t.Run("FsNewObjectNotFound", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
// Object in an existing directory
|
|
o, err := f.NewObject(ctx, "potato")
|
|
assert.Nil(t, o)
|
|
assert.Equal(t, fs.ErrorObjectNotFound, err)
|
|
// Now try an object in a nonexistent directory
|
|
o, err = f.NewObject(ctx, "directory/not/found/potato")
|
|
assert.Nil(t, o)
|
|
assert.Equal(t, fs.ErrorObjectNotFound, err)
|
|
})
|
|
|
|
// TestFsPutError tests uploading a file where there is an error
|
|
//
|
|
// It makes sure that aborting a file half way through does not create
|
|
// a file on the remote.
|
|
//
|
|
// go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutError)$'
|
|
t.Run("FsPutError", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
var N int64 = 5 * 1024
|
|
if *fstest.SizeLimit > 0 && N > *fstest.SizeLimit {
|
|
N = *fstest.SizeLimit
|
|
t.Logf("Reduce file size due to limit %d", N)
|
|
}
|
|
|
|
// Read N bytes then produce an error
|
|
contents := random.String(int(N))
|
|
buf := bytes.NewBufferString(contents)
|
|
er := &readers.ErrorReader{Err: errors.New("potato")}
|
|
in := io.MultiReader(buf, er)
|
|
|
|
obji := object.NewStaticObjectInfo(file2.Path, file2.ModTime, 2*N, true, nil, nil)
|
|
_, err := f.Put(ctx, in, obji)
|
|
// assert.Nil(t, obj) - FIXME some remotes return the object even on nil
|
|
assert.NotNil(t, err)
|
|
|
|
obj, err := f.NewObject(ctx, file2.Path)
|
|
assert.Nil(t, obj)
|
|
assert.Equal(t, fs.ErrorObjectNotFound, err)
|
|
})
|
|
|
|
t.Run("FsPutZeroLength", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
TestPutLarge(ctx, t, f, &fstest.Item{
|
|
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
|
Path: "zero-length-file",
|
|
Size: int64(0),
|
|
})
|
|
})
|
|
|
|
t.Run("FsOpenWriterAt", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
openWriterAt := f.Features().OpenWriterAt
|
|
if openWriterAt == nil {
|
|
t.Skip("FS has no OpenWriterAt interface")
|
|
}
|
|
path := "writer-at-subdir/writer-at-file"
|
|
out, err := openWriterAt(ctx, path, -1)
|
|
require.NoError(t, err)
|
|
|
|
var n int
|
|
n, err = out.WriteAt([]byte("def"), 3)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 3, n)
|
|
n, err = out.WriteAt([]byte("ghi"), 6)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 3, n)
|
|
n, err = out.WriteAt([]byte("abc"), 0)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, 3, n)
|
|
|
|
assert.NoError(t, out.Close())
|
|
|
|
obj := fstest.NewObject(ctx, t, f, path)
|
|
assert.Equal(t, "abcdefghi", ReadObject(ctx, t, obj, -1), "contents of file differ")
|
|
|
|
assert.NoError(t, obj.Remove(ctx))
|
|
assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir"))
|
|
})
|
|
|
|
// TestFsOpenChunkWriter tests writing in chunks to fs
|
|
// then reads back the contents and check if they match
|
|
// go test -v -run 'TestIntegration/FsMkdir/FsOpenChunkWriter'
|
|
t.Run("FsOpenChunkWriter", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
openChunkWriter := f.Features().OpenChunkWriter
|
|
if openChunkWriter == nil {
|
|
t.Skip("FS has no OpenChunkWriter interface")
|
|
}
|
|
size5MBs := 5 * 1024 * 1024
|
|
contents1 := random.String(size5MBs)
|
|
contents2 := random.String(size5MBs)
|
|
|
|
size1MB := 1 * 1024 * 1024
|
|
contents3 := random.String(size1MB)
|
|
|
|
path := "writer-at-subdir/writer-at-file"
|
|
objSrc := object.NewStaticObjectInfo(path+"-WRONG-REMOTE", file1.ModTime, -1, true, nil, nil)
|
|
_, out, err := openChunkWriter(ctx, path, objSrc, &fs.ChunkOption{
|
|
ChunkSize: int64(size5MBs),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
var n int64
|
|
n, err = out.WriteChunk(ctx, 1, strings.NewReader(contents2))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, int64(size5MBs), n)
|
|
n, err = out.WriteChunk(ctx, 2, strings.NewReader(contents3))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, int64(size1MB), n)
|
|
n, err = out.WriteChunk(ctx, 0, strings.NewReader(contents1))
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, int64(size5MBs), n)
|
|
|
|
assert.NoError(t, out.Close(ctx))
|
|
|
|
obj := fstest.NewObject(ctx, t, f, path)
|
|
originalContents := contents1 + contents2 + contents3
|
|
fileContents := ReadObject(ctx, t, obj, -1)
|
|
isEqual := originalContents == fileContents
|
|
assert.True(t, isEqual, "contents of file differ")
|
|
|
|
assert.NoError(t, obj.Remove(ctx))
|
|
assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir"))
|
|
})
|
|
|
|
// TestFsChangeNotify tests that changes are properly
|
|
// propagated
|
|
//
|
|
// go test -v -remote TestDrive: -run '^Test(Setup|Init|FsChangeNotify)$' -verbose
|
|
t.Run("FsChangeNotify", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
// Check have ChangeNotify
|
|
doChangeNotify := f.Features().ChangeNotify
|
|
if doChangeNotify == nil {
|
|
t.Skip("FS has no ChangeNotify interface")
|
|
}
|
|
|
|
err := operations.Mkdir(ctx, f, "dir")
|
|
require.NoError(t, err)
|
|
|
|
pollInterval := make(chan time.Duration)
|
|
dirChanges := map[string]struct{}{}
|
|
objChanges := map[string]struct{}{}
|
|
doChangeNotify(ctx, func(x string, e fs.EntryType) {
|
|
fs.Debugf(nil, "doChangeNotify(%q, %+v)", x, e)
|
|
if strings.HasPrefix(x, file1.Path[:5]) || strings.HasPrefix(x, file2.Path[:5]) {
|
|
fs.Debugf(nil, "Ignoring notify for file1 or file2: %q, %v", x, e)
|
|
return
|
|
}
|
|
if e == fs.EntryDirectory {
|
|
dirChanges[x] = struct{}{}
|
|
} else if e == fs.EntryObject {
|
|
objChanges[x] = struct{}{}
|
|
}
|
|
}, pollInterval)
|
|
defer func() { close(pollInterval) }()
|
|
pollInterval <- time.Second
|
|
|
|
var dirs []string
|
|
for _, idx := range []int{1, 3, 2} {
|
|
dir := fmt.Sprintf("dir/subdir%d", idx)
|
|
err = operations.Mkdir(ctx, f, dir)
|
|
require.NoError(t, err)
|
|
dirs = append(dirs, dir)
|
|
}
|
|
|
|
var objs []fs.Object
|
|
for _, idx := range []int{2, 4, 3} {
|
|
file := fstest.Item{
|
|
ModTime: time.Now(),
|
|
Path: fmt.Sprintf("dir/file%d", idx),
|
|
}
|
|
_, o := testPut(ctx, t, f, &file)
|
|
objs = append(objs, o)
|
|
}
|
|
|
|
// Looks for each item in wants in changes -
|
|
// if they are all found it returns true
|
|
contains := func(changes map[string]struct{}, wants []string) bool {
|
|
for _, want := range wants {
|
|
_, ok := changes[want]
|
|
if !ok {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Wait a little while for the changes to come in
|
|
wantDirChanges := []string{"dir/subdir1", "dir/subdir3", "dir/subdir2"}
|
|
wantObjChanges := []string{"dir/file2", "dir/file4", "dir/file3"}
|
|
ok := false
|
|
for tries := 1; tries < 10; tries++ {
|
|
ok = contains(dirChanges, wantDirChanges) && contains(objChanges, wantObjChanges)
|
|
if ok {
|
|
break
|
|
}
|
|
t.Logf("Try %d/10 waiting for dirChanges and objChanges", tries)
|
|
time.Sleep(3 * time.Second)
|
|
}
|
|
if !ok {
|
|
t.Errorf("%+v does not contain %+v or \n%+v does not contain %+v", dirChanges, wantDirChanges, objChanges, wantObjChanges)
|
|
}
|
|
|
|
// tidy up afterwards
|
|
for _, o := range objs {
|
|
assert.NoError(t, o.Remove(ctx))
|
|
}
|
|
dirs = append(dirs, "dir")
|
|
for _, dir := range dirs {
|
|
assert.NoError(t, f.Rmdir(ctx, dir))
|
|
}
|
|
})
|
|
|
|
// TestFsPut files writes file1, file2 and tests an update
|
|
//
|
|
// Tests that require file1, file2 are within this
|
|
t.Run("FsPutFiles", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
file1Contents, _ = testPut(ctx, t, f, &file1)
|
|
/* file2Contents = */ testPut(ctx, t, f, &file2)
|
|
file1Contents, _ = testPutMimeType(ctx, t, f, &file1, file1MimeType, file1Metadata)
|
|
// Note that the next test will check there are no duplicated file names
|
|
|
|
// TestFsListDirFile2 tests the files are correctly uploaded by doing
|
|
// Depth 1 directory listings
|
|
TestFsListDirFile2 := func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
list := func(dir string, expectedDirNames, expectedObjNames []string) {
|
|
var objNames, dirNames []string
|
|
for i := 1; i <= *fstest.ListRetries; i++ {
|
|
objs, dirs, err := walk.GetAll(ctx, f, dir, true, 1)
|
|
if errors.Is(err, fs.ErrorDirNotFound) {
|
|
objs, dirs, err = walk.GetAll(ctx, f, dir, true, 1)
|
|
}
|
|
require.NoError(t, err)
|
|
objNames = objsToNames(objs)
|
|
dirNames = dirsToNames(dirs)
|
|
if len(objNames) >= len(expectedObjNames) && len(dirNames) >= len(expectedDirNames) {
|
|
break
|
|
}
|
|
t.Logf("Sleeping for 1 second for TestFsListDirFile2 eventual consistency: %d/%d", i, *fstest.ListRetries)
|
|
time.Sleep(1 * time.Second)
|
|
}
|
|
assert.Equal(t, expectedDirNames, dirNames)
|
|
assert.Equal(t, expectedObjNames, objNames)
|
|
}
|
|
dir := file2.Path
|
|
deepest := true
|
|
for dir != "" {
|
|
expectedObjNames := []string{}
|
|
expectedDirNames := []string{}
|
|
child := dir
|
|
dir = path.Dir(dir)
|
|
if dir == "." {
|
|
dir = ""
|
|
expectedObjNames = append(expectedObjNames, file1.Path)
|
|
}
|
|
if deepest {
|
|
expectedObjNames = append(expectedObjNames, file2.Path)
|
|
deepest = false
|
|
} else {
|
|
expectedDirNames = append(expectedDirNames, child)
|
|
}
|
|
list(dir, expectedDirNames, expectedObjNames)
|
|
}
|
|
}
|
|
t.Run("FsListDirFile2", TestFsListDirFile2)
|
|
|
|
// TestFsListRDirFile2 tests the files are correctly uploaded by doing
|
|
// Depth 1 directory listings using ListR
|
|
t.Run("FsListRDirFile2", func(t *testing.T) {
|
|
defer skipIfNotListR(t)()
|
|
TestFsListDirFile2(t)
|
|
})
|
|
|
|
// Test the files are all there with walk.ListR recursive listings
|
|
t.Run("FsListR", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
objs, dirs, err := walk.GetAll(ctx, f, "", true, -1)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{
|
|
"hello? sausage",
|
|
"hello? sausage/êé",
|
|
"hello? sausage/êé/Hello, 世界",
|
|
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
|
|
}, dirsToNames(dirs))
|
|
assert.Equal(t, []string{
|
|
"file name.txt",
|
|
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠/z.txt",
|
|
}, objsToNames(objs))
|
|
})
|
|
|
|
// Test the files are all there with
|
|
// walk.ListR recursive listings on a sub dir
|
|
t.Run("FsListRSubdir", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
objs, dirs, err := walk.GetAll(ctx, f, path.Dir(path.Dir(path.Dir(path.Dir(file2.Path)))), true, -1)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{
|
|
"hello? sausage/êé",
|
|
"hello? sausage/êé/Hello, 世界",
|
|
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
|
|
}, dirsToNames(dirs))
|
|
assert.Equal(t, []string{
|
|
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠/z.txt",
|
|
}, objsToNames(objs))
|
|
})
|
|
|
|
// TestFsListDirRoot tests that DirList works in the root
|
|
TestFsListDirRoot := func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
rootRemote, err := fs.NewFs(context.Background(), remoteName)
|
|
require.NoError(t, err)
|
|
_, dirs, err := walk.GetAll(ctx, rootRemote, "", true, 1)
|
|
require.NoError(t, err)
|
|
assert.Contains(t, dirsToNames(dirs), subRemoteLeaf, "Remote leaf not found")
|
|
}
|
|
t.Run("FsListDirRoot", TestFsListDirRoot)
|
|
|
|
// TestFsListRDirRoot tests that DirList works in the root using ListR
|
|
t.Run("FsListRDirRoot", func(t *testing.T) {
|
|
defer skipIfNotListR(t)()
|
|
TestFsListDirRoot(t)
|
|
})
|
|
|
|
// TestFsListSubdir tests List works for a subdirectory
|
|
TestFsListSubdir := func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
fileName := file2.Path
|
|
var err error
|
|
var objs []fs.Object
|
|
var dirs []fs.Directory
|
|
for i := 0; i < 2; i++ {
|
|
dir, _ := path.Split(fileName)
|
|
dir = dir[:len(dir)-1]
|
|
objs, dirs, err = walk.GetAll(ctx, f, dir, true, -1)
|
|
}
|
|
require.NoError(t, err)
|
|
require.Len(t, objs, 1)
|
|
assert.Equal(t, fileName, objs[0].Remote())
|
|
require.Len(t, dirs, 0)
|
|
}
|
|
t.Run("FsListSubdir", TestFsListSubdir)
|
|
|
|
// TestFsListRSubdir tests List works for a subdirectory using ListR
|
|
t.Run("FsListRSubdir", func(t *testing.T) {
|
|
defer skipIfNotListR(t)()
|
|
TestFsListSubdir(t)
|
|
})
|
|
|
|
// TestFsListLevel2 tests List works for 2 levels
|
|
TestFsListLevel2 := func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
objs, dirs, err := walk.GetAll(ctx, f, "", true, 2)
|
|
if err == fs.ErrorLevelNotSupported {
|
|
return
|
|
}
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{file1.Path}, objsToNames(objs))
|
|
assert.Equal(t, []string{"hello? sausage", "hello? sausage/êé"}, dirsToNames(dirs))
|
|
}
|
|
t.Run("FsListLevel2", TestFsListLevel2)
|
|
|
|
// TestFsListRLevel2 tests List works for 2 levels using ListR
|
|
t.Run("FsListRLevel2", func(t *testing.T) {
|
|
defer skipIfNotListR(t)()
|
|
TestFsListLevel2(t)
|
|
})
|
|
|
|
// TestFsListFile1 tests file present
|
|
t.Run("FsListFile1", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
fstest.CheckListing(t, f, []fstest.Item{file1, file2})
|
|
})
|
|
|
|
// TestFsNewObject tests NewObject
|
|
t.Run("FsNewObject", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
file1.Check(t, obj, f.Precision())
|
|
})
|
|
|
|
// FsNewObjectCaseInsensitive tests NewObject on a case insensitive file system
|
|
t.Run("FsNewObjectCaseInsensitive", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
if !f.Features().CaseInsensitive {
|
|
t.Skip("Not Case Insensitive")
|
|
}
|
|
obj := fstest.NewObject(ctx, t, f, toUpperASCII(file1.Path))
|
|
file1.Check(t, obj, f.Precision())
|
|
t.Run("Dir", func(t *testing.T) {
|
|
obj := fstest.NewObject(ctx, t, f, toUpperASCII(file2.Path))
|
|
file2.Check(t, obj, f.Precision())
|
|
})
|
|
})
|
|
|
|
// TestFsListFile1and2 tests two files present
|
|
t.Run("FsListFile1and2", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
fstest.CheckListing(t, f, []fstest.Item{file1, file2})
|
|
})
|
|
|
|
// TestFsNewObjectDir tests NewObject on a directory which should produce fs.ErrorIsDir if possible or fs.ErrorObjectNotFound if not
|
|
t.Run("FsNewObjectDir", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
dir := path.Dir(file2.Path)
|
|
obj, err := f.NewObject(ctx, dir)
|
|
assert.Nil(t, obj)
|
|
assert.True(t, err == fs.ErrorIsDir || err == fs.ErrorObjectNotFound, fmt.Sprintf("Wrong error: expecting fs.ErrorIsDir or fs.ErrorObjectNotFound but got: %#v", err))
|
|
})
|
|
|
|
// TestFsPurge tests Purge
|
|
t.Run("FsPurge", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
// Check have Purge
|
|
doPurge := f.Features().Purge
|
|
if doPurge == nil {
|
|
t.Skip("FS has no Purge interface")
|
|
}
|
|
|
|
// put up a file to purge
|
|
fileToPurge := fstest.Item{
|
|
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
|
Path: "dirToPurge/fileToPurge.txt",
|
|
}
|
|
_, _ = testPut(ctx, t, f, &fileToPurge)
|
|
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2, fileToPurge}, []string{
|
|
"dirToPurge",
|
|
"hello? sausage",
|
|
"hello? sausage/êé",
|
|
"hello? sausage/êé/Hello, 世界",
|
|
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
|
|
}, fs.GetModifyWindow(ctx, f))
|
|
|
|
// Now purge it
|
|
err = operations.Purge(ctx, f, "dirToPurge")
|
|
require.NoError(t, err)
|
|
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2}, []string{
|
|
"hello? sausage",
|
|
"hello? sausage/êé",
|
|
"hello? sausage/êé/Hello, 世界",
|
|
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
|
|
}, fs.GetModifyWindow(ctx, f))
|
|
})
|
|
|
|
// TestFsPurge tests Purge on the Root
|
|
t.Run("FsPurgeRoot", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
// Check have Purge
|
|
doPurge := f.Features().Purge
|
|
if doPurge == nil {
|
|
t.Skip("FS has no Purge interface")
|
|
}
|
|
|
|
// put up a file to purge
|
|
fileToPurge := fstest.Item{
|
|
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
|
Path: "dirToPurgeFromRoot/fileToPurgeFromRoot.txt",
|
|
}
|
|
_, _ = testPut(ctx, t, f, &fileToPurge)
|
|
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2, fileToPurge}, []string{
|
|
"dirToPurgeFromRoot",
|
|
"hello? sausage",
|
|
"hello? sausage/êé",
|
|
"hello? sausage/êé/Hello, 世界",
|
|
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
|
|
}, fs.GetModifyWindow(ctx, f))
|
|
|
|
// Create a new Fs pointing at the directory
|
|
remoteName := subRemoteName + "/" + "dirToPurgeFromRoot"
|
|
fPurge, err := fs.NewFs(context.Background(), remoteName)
|
|
require.NoError(t, err)
|
|
|
|
// Now purge it from the root
|
|
err = operations.Purge(ctx, fPurge, "")
|
|
require.NoError(t, err)
|
|
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2}, []string{
|
|
"hello? sausage",
|
|
"hello? sausage/êé",
|
|
"hello? sausage/êé/Hello, 世界",
|
|
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
|
|
}, fs.GetModifyWindow(ctx, f))
|
|
})
|
|
|
|
// TestFsListRootedSubdir tests putting and listing with an Fs that is rooted at a subdirectory 2 levels down
|
|
TestFsListRootedSubdir := func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
newF, err := cache.Get(ctx, subRemoteName+"/hello? sausage/êé")
|
|
assert.NoError(t, err)
|
|
nestedFile := fstest.Item{
|
|
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
|
Path: "a/b/c/d/e.txt",
|
|
}
|
|
_, _ = testPut(ctx, t, newF, &nestedFile)
|
|
|
|
objs, dirs, err := walk.GetAll(ctx, newF, "", true, 10)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{`Hello, 世界/ " ' @ < > & ? + ≠/z.txt`, nestedFile.Path}, objsToNames(objs))
|
|
assert.Equal(t, []string{`Hello, 世界`, `Hello, 世界/ " ' @ < > & ? + ≠`, "a", "a/b", "a/b/c", "a/b/c/d"}, dirsToNames(dirs))
|
|
|
|
// cleanup
|
|
err = operations.Purge(ctx, newF, "a")
|
|
require.NoError(t, err)
|
|
}
|
|
t.Run("FsListRootedSubdir", TestFsListRootedSubdir)
|
|
|
|
// TestFsCopy tests Copy
|
|
t.Run("FsCopy", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
// Check have Copy
|
|
doCopy := f.Features().Copy
|
|
if doCopy == nil {
|
|
t.Skip("FS has no Copier interface")
|
|
}
|
|
|
|
// Test with file2 so have + and ' ' in file name
|
|
var file2Copy = file2
|
|
file2Copy.Path += "-copy"
|
|
|
|
// do the copy
|
|
src := fstest.NewObject(ctx, t, f, file2.Path)
|
|
dst, err := doCopy(ctx, src, file2Copy.Path)
|
|
if err == fs.ErrorCantCopy {
|
|
t.Skip("FS can't copy")
|
|
}
|
|
require.NoError(t, err, fmt.Sprintf("Error: %#v", err))
|
|
|
|
// check file exists in new listing
|
|
fstest.CheckListing(t, f, []fstest.Item{file1, file2, file2Copy})
|
|
|
|
// Check dst lightly - list above has checked ModTime/Hashes
|
|
assert.Equal(t, file2Copy.Path, dst.Remote())
|
|
|
|
// check that mutating dst does not mutate src
|
|
err = dst.SetModTime(ctx, fstest.Time("2004-03-03T04:05:06.499999999Z"))
|
|
if err != fs.ErrorCantSetModTimeWithoutDelete && err != fs.ErrorCantSetModTime {
|
|
assert.NoError(t, err)
|
|
assert.False(t, src.ModTime(ctx).Equal(dst.ModTime(ctx)), "mutating dst should not mutate src -- is it Copying by pointer?")
|
|
}
|
|
|
|
// Delete copy
|
|
err = dst.Remove(ctx)
|
|
require.NoError(t, err)
|
|
|
|
// Test that server side copying files does the correct thing with metadata
|
|
t.Run("Metadata", func(t *testing.T) {
|
|
if !f.Features().WriteMetadata {
|
|
t.Skip("Skipping test as can't write metadata")
|
|
}
|
|
ctx, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
|
|
// Create file with metadata
|
|
const srcName = "test metadata copy.txt"
|
|
const dstName = "test metadata copied.txt"
|
|
t1 := fstest.Time("2003-02-03T04:05:06.499999999Z")
|
|
t2 := fstest.Time("2004-03-03T04:05:06.499999999Z")
|
|
fileSrc := fstest.NewItem(srcName, srcName, t1)
|
|
contents := random.String(100)
|
|
var testMetadata = fs.Metadata{
|
|
// System metadata supported by all backends
|
|
"mtime": t1.Format(time.RFC3339Nano),
|
|
// User metadata
|
|
"potato": "jersey",
|
|
}
|
|
oSrc := PutTestContentsMetadata(ctx, t, f, &fileSrc, contents, true, "text/plain", testMetadata)
|
|
fstest.CheckEntryMetadata(ctx, t, f, oSrc, testMetadata)
|
|
|
|
// Copy it with --metadata-set
|
|
ci.MetadataSet = fs.Metadata{
|
|
// System metadata supported by all backends
|
|
"mtime": t2.Format(time.RFC3339Nano),
|
|
// User metadata
|
|
"potato": "royal",
|
|
}
|
|
oDst, err := doCopy(ctx, oSrc, dstName)
|
|
require.NoError(t, err)
|
|
fileDst := fileSrc
|
|
fileDst.Path = dstName
|
|
fileDst.ModTime = t2
|
|
fstest.CheckListing(t, f, []fstest.Item{file1, file2, fileSrc, fileDst})
|
|
|
|
// Check metadata is correct
|
|
fstest.CheckEntryMetadata(ctx, t, f, oDst, ci.MetadataSet)
|
|
oDst = fstest.NewObject(ctx, t, f, dstName)
|
|
fstest.CheckEntryMetadata(ctx, t, f, oDst, ci.MetadataSet)
|
|
|
|
// Remove test files
|
|
require.NoError(t, oSrc.Remove(ctx))
|
|
require.NoError(t, oDst.Remove(ctx))
|
|
})
|
|
})
|
|
|
|
// TestFsMove tests Move
|
|
t.Run("FsMove", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
// Check have Move
|
|
doMove := f.Features().Move
|
|
if doMove == nil {
|
|
t.Skip("FS has no Mover interface")
|
|
}
|
|
|
|
// state of files now:
|
|
// 1: file name.txt
|
|
// 2: hello sausage?/../z.txt
|
|
|
|
var file1Move = file1
|
|
var file2Move = file2
|
|
|
|
// check happy path, i.e. no naming conflicts when rename and move are two
|
|
// separate operations
|
|
file2Move.Path = "other.txt"
|
|
src := fstest.NewObject(ctx, t, f, file2.Path)
|
|
dst, err := doMove(ctx, src, file2Move.Path)
|
|
if err == fs.ErrorCantMove {
|
|
t.Skip("FS can't move")
|
|
}
|
|
require.NoError(t, err)
|
|
// check file exists in new listing
|
|
fstest.CheckListing(t, f, []fstest.Item{file1, file2Move})
|
|
// Check dst lightly - list above has checked ModTime/Hashes
|
|
assert.Equal(t, file2Move.Path, dst.Remote())
|
|
// 1: file name.txt
|
|
// 2: other.txt
|
|
|
|
// Check conflict on "rename, then move"
|
|
file1Move.Path = "moveTest/other.txt"
|
|
src = fstest.NewObject(ctx, t, f, file1.Path)
|
|
_, err = doMove(ctx, src, file1Move.Path)
|
|
require.NoError(t, err)
|
|
fstest.CheckListing(t, f, []fstest.Item{file1Move, file2Move})
|
|
// 1: moveTest/other.txt
|
|
// 2: other.txt
|
|
|
|
// Check conflict on "move, then rename"
|
|
src = fstest.NewObject(ctx, t, f, file1Move.Path)
|
|
_, err = doMove(ctx, src, file1.Path)
|
|
require.NoError(t, err)
|
|
fstest.CheckListing(t, f, []fstest.Item{file1, file2Move})
|
|
// 1: file name.txt
|
|
// 2: other.txt
|
|
|
|
src = fstest.NewObject(ctx, t, f, file2Move.Path)
|
|
_, err = doMove(ctx, src, file2.Path)
|
|
require.NoError(t, err)
|
|
fstest.CheckListing(t, f, []fstest.Item{file1, file2})
|
|
// 1: file name.txt
|
|
// 2: hello sausage?/../z.txt
|
|
|
|
// Tidy up moveTest directory
|
|
require.NoError(t, f.Rmdir(ctx, "moveTest"))
|
|
|
|
// Test that server side moving files does the correct thing with metadata
|
|
t.Run("Metadata", func(t *testing.T) {
|
|
if !f.Features().WriteMetadata {
|
|
t.Skip("Skipping test as can't write metadata")
|
|
}
|
|
ctx, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
|
|
// Create file with metadata
|
|
const name = "test metadata move.txt"
|
|
const newName = "test metadata moved.txt"
|
|
t1 := fstest.Time("2003-02-03T04:05:06.499999999Z")
|
|
t2 := fstest.Time("2004-03-03T04:05:06.499999999Z")
|
|
file := fstest.NewItem(name, name, t1)
|
|
contents := random.String(100)
|
|
var testMetadata = fs.Metadata{
|
|
// System metadata supported by all backends
|
|
"mtime": t1.Format(time.RFC3339Nano),
|
|
// User metadata
|
|
"potato": "jersey",
|
|
}
|
|
o := PutTestContentsMetadata(ctx, t, f, &file, contents, true, "text/plain", testMetadata)
|
|
fstest.CheckEntryMetadata(ctx, t, f, o, testMetadata)
|
|
|
|
// Move it with --metadata-set
|
|
ci.MetadataSet = fs.Metadata{
|
|
// System metadata supported by all backends
|
|
"mtime": t2.Format(time.RFC3339Nano),
|
|
// User metadata
|
|
"potato": "royal",
|
|
}
|
|
newO, err := doMove(ctx, o, newName)
|
|
require.NoError(t, err)
|
|
file.Path = newName
|
|
file.ModTime = t2
|
|
fstest.CheckListing(t, f, []fstest.Item{file1, file2, file})
|
|
|
|
// Check metadata is correct
|
|
fstest.CheckEntryMetadata(ctx, t, f, newO, ci.MetadataSet)
|
|
newO = fstest.NewObject(ctx, t, f, newName)
|
|
fstest.CheckEntryMetadata(ctx, t, f, newO, ci.MetadataSet)
|
|
|
|
// Remove test file
|
|
require.NoError(t, newO.Remove(ctx))
|
|
})
|
|
})
|
|
|
|
// Move src to this remote using server-side move operations.
|
|
//
|
|
// Will only be called if src.Fs().Name() == f.Name()
|
|
//
|
|
// If it isn't possible then return fs.ErrorCantDirMove
|
|
//
|
|
// If destination exists then return fs.ErrorDirExists
|
|
|
|
// TestFsDirMove tests DirMove
|
|
//
|
|
// go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|FsDirMove)$
|
|
t.Run("FsDirMove", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
// Check have DirMove
|
|
doDirMove := f.Features().DirMove
|
|
if doDirMove == nil {
|
|
t.Skip("FS has no DirMover interface")
|
|
}
|
|
|
|
// Check it can't move onto itself
|
|
err := doDirMove(ctx, f, "", "")
|
|
require.Equal(t, fs.ErrorDirExists, err)
|
|
|
|
// new remote
|
|
newRemote, _, removeNewRemote, err := fstest.RandomRemote()
|
|
require.NoError(t, err)
|
|
defer removeNewRemote()
|
|
|
|
const newName = "new_name/sub_new_name"
|
|
// try the move
|
|
err = newRemote.Features().DirMove(ctx, f, "", newName)
|
|
require.NoError(t, err)
|
|
|
|
// check remotes
|
|
// remote should not exist here
|
|
_, err = f.List(ctx, "")
|
|
assert.True(t, errors.Is(err, fs.ErrorDirNotFound))
|
|
//fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, remote.Precision())
|
|
file1Copy := file1
|
|
file1Copy.Path = path.Join(newName, file1.Path)
|
|
file2Copy := file2
|
|
file2Copy.Path = path.Join(newName, file2.Path)
|
|
fstest.CheckListingWithPrecision(t, newRemote, []fstest.Item{file2Copy, file1Copy}, []string{
|
|
"new_name",
|
|
"new_name/sub_new_name",
|
|
"new_name/sub_new_name/hello? sausage",
|
|
"new_name/sub_new_name/hello? sausage/êé",
|
|
"new_name/sub_new_name/hello? sausage/êé/Hello, 世界",
|
|
"new_name/sub_new_name/hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
|
|
}, newRemote.Precision())
|
|
|
|
// move it back
|
|
err = doDirMove(ctx, newRemote, newName, "")
|
|
require.NoError(t, err)
|
|
|
|
// check remotes
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{file2, file1}, []string{
|
|
"hello? sausage",
|
|
"hello? sausage/êé",
|
|
"hello? sausage/êé/Hello, 世界",
|
|
"hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠",
|
|
}, f.Precision())
|
|
fstest.CheckListingWithPrecision(t, newRemote, []fstest.Item{}, []string{
|
|
"new_name",
|
|
}, newRemote.Precision())
|
|
})
|
|
|
|
// TestFsRmdirFull tests removing a non empty directory
|
|
t.Run("FsRmdirFull", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
if isBucketBasedButNotRoot(f) {
|
|
t.Skip("Skipping test as non root bucket-based remote")
|
|
}
|
|
err := f.Rmdir(ctx, "")
|
|
require.Error(t, err, "Expecting error on RMdir on non empty remote")
|
|
})
|
|
|
|
// TestFsPrecision tests the Precision of the Fs
|
|
t.Run("FsPrecision", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
precision := f.Precision()
|
|
if precision == fs.ModTimeNotSupported {
|
|
return
|
|
}
|
|
if precision > time.Second || precision < 0 {
|
|
t.Fatalf("Precision out of range %v", precision)
|
|
}
|
|
// FIXME check expected precision
|
|
})
|
|
|
|
// TestObjectString tests the Object String method
|
|
t.Run("ObjectString", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
assert.Equal(t, file1.Path, obj.String())
|
|
if opt.NilObject != nil {
|
|
assert.Equal(t, "<nil>", opt.NilObject.String())
|
|
}
|
|
})
|
|
|
|
// TestObjectFs tests the object can be found
|
|
t.Run("ObjectFs", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
// If this is set we don't do the direct comparison of
|
|
// the Fs from the object as it may be different
|
|
if opt.SkipFsMatch {
|
|
return
|
|
}
|
|
testRemote := f
|
|
if obj.Fs() != testRemote {
|
|
// Check to see if this wraps something else
|
|
if doUnWrap := testRemote.Features().UnWrap; doUnWrap != nil {
|
|
testRemote = doUnWrap()
|
|
}
|
|
}
|
|
assert.Equal(t, obj.Fs(), testRemote)
|
|
})
|
|
|
|
// TestObjectRemote tests the Remote is correct
|
|
t.Run("ObjectRemote", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
assert.Equal(t, file1.Path, obj.Remote())
|
|
})
|
|
|
|
// TestObjectHashes checks all the hashes the object supports
|
|
t.Run("ObjectHashes", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
file1.CheckHashes(t, obj)
|
|
})
|
|
|
|
// TestObjectModTime tests the ModTime of the object is correct
|
|
TestObjectModTime := func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision())
|
|
}
|
|
t.Run("ObjectModTime", TestObjectModTime)
|
|
|
|
// TestObjectMimeType tests the MimeType of the object is correct
|
|
t.Run("ObjectMimeType", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
features := f.Features()
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
do, ok := obj.(fs.MimeTyper)
|
|
if !ok {
|
|
require.False(t, features.ReadMimeType, "Features.ReadMimeType is set but Object.MimeType method not found")
|
|
t.Skip("MimeType method not supported")
|
|
}
|
|
mimeType := do.MimeType(ctx)
|
|
if !features.ReadMimeType {
|
|
require.Equal(t, "", mimeType, "Features.ReadMimeType is not set but Object.MimeType returned a non-empty MimeType")
|
|
} else if features.WriteMimeType {
|
|
assert.Equal(t, file1MimeType, mimeType, "can read and write mime types but failed")
|
|
} else {
|
|
if strings.ContainsRune(mimeType, ';') {
|
|
assert.Equal(t, "text/plain; charset=utf-8", mimeType)
|
|
} else {
|
|
assert.Equal(t, "text/plain", mimeType)
|
|
}
|
|
}
|
|
})
|
|
|
|
// TestObjectMetadata tests the Metadata of the object is correct
|
|
t.Run("ObjectMetadata", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
ctx, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
features := f.Features()
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
do, objectHasMetadata := obj.(fs.Metadataer)
|
|
if objectHasMetadata || features.ReadMetadata || features.WriteMetadata || features.UserMetadata {
|
|
fsInfo := fs.FindFromFs(f)
|
|
require.NotNil(t, fsInfo)
|
|
require.NotNil(t, fsInfo.MetadataInfo, "Object declares metadata support but no MetadataInfo in RegInfo")
|
|
}
|
|
if !objectHasMetadata {
|
|
require.False(t, features.ReadMetadata, "Features.ReadMetadata is set but Object.Metadata method not found")
|
|
t.Skip("Metadata method not supported")
|
|
}
|
|
metadata, err := do.Metadata(ctx)
|
|
require.NoError(t, err)
|
|
// check standard metadata
|
|
for k, v := range metadata {
|
|
switch k {
|
|
case "atime", "btime", "mtime":
|
|
mtime, err := time.Parse(time.RFC3339Nano, v)
|
|
require.NoError(t, err)
|
|
if k == "mtime" {
|
|
fstest.AssertTimeEqualWithPrecision(t, file1.Path, file1.ModTime, mtime, f.Precision())
|
|
}
|
|
}
|
|
}
|
|
if !features.ReadMetadata {
|
|
if metadata != nil && !features.Overlay {
|
|
require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata: %#v", metadata)
|
|
}
|
|
} else if features.WriteMetadata {
|
|
require.NotNil(t, metadata)
|
|
if features.UserMetadata {
|
|
// check all the metadata bits we uploaded are present - there may be more we didn't write
|
|
for k, v := range file1Metadata {
|
|
assert.Equal(t, v, metadata[k], "can read and write metadata but failed on key %q (want=%+v, got=%+v)", k, file1Metadata, metadata)
|
|
}
|
|
}
|
|
// Now test we can set the mtime and content-type via the metadata and these take precedence
|
|
t.Run("mtime", func(t *testing.T) {
|
|
path := "metadatatest"
|
|
mtimeModTime := fstest.Time("2002-02-03T04:05:06.499999999Z")
|
|
modTime := fstest.Time("2003-02-03T04:05:06.499999999Z")
|
|
item := fstest.NewItem(path, path, modTime)
|
|
metaMimeType := "application/zip"
|
|
mimeType := "application/gzip"
|
|
metadata := fs.Metadata{
|
|
"mtime": mtimeModTime.Format(time.RFC3339Nano),
|
|
"content-type": metaMimeType,
|
|
}
|
|
// This checks the mtime is correct also and returns the re-read object
|
|
_, obj := testPutMimeType(ctx, t, f, &item, mimeType, metadata)
|
|
defer func() {
|
|
assert.NoError(t, obj.Remove(ctx))
|
|
}()
|
|
// Check content-type got updated too
|
|
if features.ReadMimeType && features.WriteMimeType {
|
|
// read the object from scratch
|
|
o, err := f.NewObject(ctx, path)
|
|
require.NoError(t, err)
|
|
|
|
// Check the mimetype is correct
|
|
do, ok := o.(fs.MimeTyper)
|
|
require.True(t, ok)
|
|
gotMimeType := do.MimeType(ctx)
|
|
assert.Equal(t, metaMimeType, gotMimeType)
|
|
}
|
|
})
|
|
} // else: Have some metadata here we didn't write - can't really check it!
|
|
})
|
|
|
|
// TestObjectSetMetadata tests the SetMetadata of the object
|
|
t.Run("ObjectSetMetadata", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
ctx, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
features := f.Features()
|
|
|
|
// Test to see if SetMetadata is supported on an existing object before creating a new one
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
_, objectHasSetMetadata := obj.(fs.SetMetadataer)
|
|
if !objectHasSetMetadata {
|
|
t.Skip("SetMetadata method not supported")
|
|
}
|
|
if !features.Overlay {
|
|
require.True(t, features.WriteMetadata, "Features.WriteMetadata is false but Object.SetMetadata found")
|
|
}
|
|
if !features.ReadMetadata {
|
|
t.Skip("SetMetadata can't be tested without ReadMetadata")
|
|
}
|
|
|
|
// Create file with metadata
|
|
const fileName = "test set metadata.txt"
|
|
t1 := fstest.Time("2003-02-03T04:05:06.499999999Z")
|
|
t2 := fstest.Time("2004-03-03T04:05:06.499999999Z")
|
|
contents := random.String(100)
|
|
file := fstest.NewItem(fileName, contents, t1)
|
|
var testMetadata = fs.Metadata{
|
|
// System metadata supported by all backends
|
|
"mtime": t1.Format(time.RFC3339Nano),
|
|
// User metadata
|
|
"potato": "jersey",
|
|
}
|
|
obj = PutTestContentsMetadata(ctx, t, f, &file, contents, true, "text/plain", testMetadata)
|
|
fstest.CheckEntryMetadata(ctx, t, f, obj, testMetadata)
|
|
do, objectHasSetMetadata := obj.(fs.SetMetadataer)
|
|
require.True(t, objectHasSetMetadata)
|
|
|
|
// Set new metadata
|
|
err := do.SetMetadata(ctx, fs.Metadata{
|
|
// System metadata supported by all backends
|
|
"mtime": t2.Format(time.RFC3339Nano),
|
|
// User metadata
|
|
"potato": "royal",
|
|
})
|
|
if err == fs.ErrorNotImplemented {
|
|
t.Log("SetMetadata returned fs.ErrorNotImplemented")
|
|
} else {
|
|
require.NoError(t, err)
|
|
file.ModTime = t2
|
|
fstest.CheckListing(t, f, []fstest.Item{file1, file2, file})
|
|
|
|
// Check metadata is correct
|
|
fstest.CheckEntryMetadata(ctx, t, f, obj, ci.MetadataSet)
|
|
obj = fstest.NewObject(ctx, t, f, fileName)
|
|
fstest.CheckEntryMetadata(ctx, t, f, obj, ci.MetadataSet)
|
|
}
|
|
|
|
// Remove test file
|
|
require.NoError(t, obj.Remove(ctx))
|
|
})
|
|
|
|
// TestObjectSetModTime tests that SetModTime works
|
|
t.Run("ObjectSetModTime", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z")
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
err := obj.SetModTime(ctx, newModTime)
|
|
if err == fs.ErrorCantSetModTime || err == fs.ErrorCantSetModTimeWithoutDelete {
|
|
t.Log(err)
|
|
return
|
|
}
|
|
require.NoError(t, err)
|
|
file1.ModTime = newModTime
|
|
file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision())
|
|
// And make a new object and read it from there too
|
|
TestObjectModTime(t)
|
|
})
|
|
|
|
// TestObjectSize tests that Size works
|
|
t.Run("ObjectSize", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
assert.Equal(t, file1.Size, obj.Size())
|
|
})
|
|
|
|
// TestObjectOpen tests that Open works
|
|
t.Run("ObjectOpen", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
assert.Equal(t, file1Contents, ReadObject(ctx, t, obj, -1), "contents of file1 differ")
|
|
})
|
|
|
|
// TestObjectOpenSeek tests that Open works with SeekOption
|
|
t.Run("ObjectOpenSeek", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
assert.Equal(t, file1Contents[50:], ReadObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek")
|
|
})
|
|
|
|
// TestObjectOpenRange tests that Open works with RangeOption
|
|
//
|
|
// go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|ObjectOpenRange)$'
|
|
t.Run("ObjectOpenRange", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
for _, test := range []struct {
|
|
ro fs.RangeOption
|
|
wantStart, wantEnd int
|
|
}{
|
|
{fs.RangeOption{Start: 5, End: 15}, 5, 16},
|
|
{fs.RangeOption{Start: 80, End: -1}, 80, 100},
|
|
{fs.RangeOption{Start: 81, End: 100000}, 81, 100},
|
|
{fs.RangeOption{Start: -1, End: 20}, 80, 100}, // if start is omitted this means get the final bytes
|
|
// {fs.RangeOption{Start: -1, End: -1}, 0, 100}, - this seems to work but the RFC doesn't define it
|
|
} {
|
|
got := ReadObject(ctx, t, obj, -1, &test.ro)
|
|
foundAt := strings.Index(file1Contents, got)
|
|
help := fmt.Sprintf("%#v failed want [%d:%d] got [%d:%d]", test.ro, test.wantStart, test.wantEnd, foundAt, foundAt+len(got))
|
|
assert.Equal(t, file1Contents[test.wantStart:test.wantEnd], got, help)
|
|
}
|
|
})
|
|
|
|
// TestObjectPartialRead tests that reading only part of the object does the correct thing
|
|
t.Run("ObjectPartialRead", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
assert.Equal(t, file1Contents[:50], ReadObject(ctx, t, obj, 50), "contents of file1 differ after limited read")
|
|
})
|
|
|
|
// TestObjectUpdate tests that Update works
|
|
t.Run("ObjectUpdate", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
contents := random.String(200)
|
|
var h *hash.MultiHasher
|
|
|
|
file1.Size = int64(len(contents))
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
remoteBefore := obj.Remote()
|
|
obji := object.NewStaticObjectInfo(file1.Path+"-should-be-ignored.bin", file1.ModTime, int64(len(contents)), true, nil, obj.Fs())
|
|
retry(t, "Update object", func() error {
|
|
buf := bytes.NewBufferString(contents)
|
|
h = hash.NewMultiHasher()
|
|
in := io.TeeReader(buf, h)
|
|
return obj.Update(ctx, in, obji)
|
|
})
|
|
remoteAfter := obj.Remote()
|
|
assert.Equal(t, remoteBefore, remoteAfter, "Remote should not change")
|
|
file1.Hashes = h.Sums()
|
|
|
|
// check the object has been updated
|
|
file1.Check(t, obj, f.Precision())
|
|
|
|
// Re-read the object and check again
|
|
obj = fstest.NewObject(ctx, t, f, file1.Path)
|
|
file1.Check(t, obj, f.Precision())
|
|
|
|
// check contents correct
|
|
assert.Equal(t, contents, ReadObject(ctx, t, obj, -1), "contents of updated file1 differ")
|
|
file1Contents = contents
|
|
})
|
|
|
|
// TestObjectStorable tests that Storable works
|
|
t.Run("ObjectStorable", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
require.NotNil(t, !obj.Storable(), "Expecting object to be storable")
|
|
})
|
|
|
|
// TestFsIsFile tests that an error is returned along with a valid fs
|
|
// which points to the parent directory.
|
|
t.Run("FsIsFile", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
remoteName := subRemoteName + "/" + file2.Path
|
|
file2Copy := file2
|
|
file2Copy.Path = "z.txt"
|
|
fileRemote, err := fs.NewFs(context.Background(), remoteName)
|
|
require.NotNil(t, fileRemote)
|
|
assert.Equal(t, fs.ErrorIsFile, err)
|
|
|
|
// Check Fs.Root returns the right thing
|
|
t.Run("FsRoot", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
got := fileRemote.Root()
|
|
remoteDir := path.Dir(remoteName)
|
|
want := remoteDir
|
|
colon := strings.LastIndex(want, ":")
|
|
if colon >= 0 {
|
|
want = want[colon+1:]
|
|
}
|
|
if isLocalRemote {
|
|
// only check last path element on local
|
|
require.Equal(t, filepath.Base(remoteDir), filepath.Base(got))
|
|
} else {
|
|
require.Equal(t, want, got)
|
|
}
|
|
})
|
|
|
|
if strings.HasPrefix(remoteName, "TestChunker") && strings.Contains(remoteName, "Nometa") {
|
|
// TODO fix chunker and remove this bypass
|
|
t.Logf("Skip listing check -- chunker can't yet handle this tricky case")
|
|
return
|
|
}
|
|
fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
|
|
})
|
|
|
|
// TestFsIsFileNotFound tests that an error is not returned if no object is found
|
|
t.Run("FsIsFileNotFound", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
remoteName := subRemoteName + "/not found.txt"
|
|
fileRemote, err := fs.NewFs(context.Background(), remoteName)
|
|
require.NoError(t, err)
|
|
fstest.CheckListing(t, fileRemote, []fstest.Item{})
|
|
})
|
|
|
|
// Test that things work from the root
|
|
t.Run("FromRoot", func(t *testing.T) {
|
|
if features := f.Features(); features.BucketBased && !features.BucketBasedRootOK {
|
|
t.Skip("Can't list from root on this remote")
|
|
}
|
|
|
|
parsed, err := fspath.Parse(subRemoteName)
|
|
require.NoError(t, err)
|
|
configName, configLeaf := parsed.ConfigString, parsed.Path
|
|
if configName == "" {
|
|
configName, configLeaf = path.Split(subRemoteName)
|
|
} else {
|
|
configName += ":"
|
|
}
|
|
t.Logf("Opening root remote %q path %q from %q", configName, configLeaf, subRemoteName)
|
|
rootRemote, err := fs.NewFs(context.Background(), configName)
|
|
require.NoError(t, err)
|
|
|
|
file1Root := file1
|
|
file1Root.Path = path.Join(configLeaf, file1Root.Path)
|
|
file2Root := file2
|
|
file2Root.Path = path.Join(configLeaf, file2Root.Path)
|
|
var dirs []string
|
|
dir := file2.Path
|
|
for {
|
|
dir = path.Dir(dir)
|
|
if dir == "" || dir == "." || dir == "/" {
|
|
break
|
|
}
|
|
dirs = append(dirs, path.Join(configLeaf, dir))
|
|
}
|
|
|
|
// Check that we can see file1 and file2 from the root
|
|
t.Run("List", func(t *testing.T) {
|
|
fstest.CheckListingWithRoot(t, rootRemote, configLeaf, []fstest.Item{file1Root, file2Root}, dirs, rootRemote.Precision())
|
|
})
|
|
|
|
// Check that listing the entries is OK
|
|
t.Run("ListEntries", func(t *testing.T) {
|
|
entries, err := rootRemote.List(context.Background(), configLeaf)
|
|
require.NoError(t, err)
|
|
fstest.CompareItems(t, entries, []fstest.Item{file1Root}, dirs[len(dirs)-1:], rootRemote.Precision(), "ListEntries")
|
|
})
|
|
|
|
// List the root with ListR
|
|
t.Run("ListR", func(t *testing.T) {
|
|
doListR := rootRemote.Features().ListR
|
|
if doListR == nil {
|
|
t.Skip("FS has no ListR interface")
|
|
}
|
|
file1Found, file2Found := false, false
|
|
stopTime := time.Now().Add(10 * time.Second)
|
|
errTooMany := errors.New("too many files")
|
|
errFound := errors.New("found")
|
|
err := doListR(context.Background(), "", func(entries fs.DirEntries) error {
|
|
for _, entry := range entries {
|
|
remote := entry.Remote()
|
|
if remote == file1Root.Path {
|
|
file1Found = true
|
|
}
|
|
if remote == file2Root.Path {
|
|
file2Found = true
|
|
}
|
|
if file1Found && file2Found {
|
|
return errFound
|
|
}
|
|
}
|
|
if time.Now().After(stopTime) {
|
|
return errTooMany
|
|
}
|
|
return nil
|
|
})
|
|
if !errors.Is(err, errFound) && !errors.Is(err, errTooMany) {
|
|
assert.NoError(t, err)
|
|
}
|
|
if !errors.Is(err, errTooMany) {
|
|
assert.True(t, file1Found, "file1Root %q not found", file1Root.Path)
|
|
assert.True(t, file2Found, "file2Root %q not found", file2Root.Path)
|
|
} else {
|
|
t.Logf("Too many files to list - giving up")
|
|
}
|
|
})
|
|
|
|
// Create a new file
|
|
t.Run("Put", func(t *testing.T) {
|
|
file3Root := fstest.Item{
|
|
ModTime: time.Now(),
|
|
Path: path.Join(configLeaf, "created from root.txt"),
|
|
}
|
|
_, file3Obj := testPut(ctx, t, rootRemote, &file3Root)
|
|
fstest.CheckListingWithRoot(t, rootRemote, configLeaf, []fstest.Item{file1Root, file2Root, file3Root}, nil, rootRemote.Precision())
|
|
|
|
// And then remove it
|
|
t.Run("Remove", func(t *testing.T) {
|
|
require.NoError(t, file3Obj.Remove(context.Background()))
|
|
fstest.CheckListingWithRoot(t, rootRemote, configLeaf, []fstest.Item{file1Root, file2Root}, nil, rootRemote.Precision())
|
|
})
|
|
})
|
|
})
|
|
|
|
// TestPublicLink tests creation of sharable, public links
|
|
// go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|PublicLink)$'
|
|
t.Run("PublicLink", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
publicLinkFunc := f.Features().PublicLink
|
|
if publicLinkFunc == nil {
|
|
t.Skip("FS has no PublicLinker interface")
|
|
}
|
|
|
|
type PublicLinkFunc func(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error)
|
|
wrapPublicLinkFunc := func(f PublicLinkFunc) PublicLinkFunc {
|
|
return func(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
|
link, err = publicLinkFunc(ctx, remote, expire, unlink)
|
|
if err == nil {
|
|
return
|
|
}
|
|
// For OneDrive Personal, link expiry is a premium feature
|
|
// Don't let it fail the test (https://github.com/rclone/rclone/issues/5420)
|
|
if fsInfo.Name == "onedrive" && strings.Contains(err.Error(), "accountUpgradeRequired") {
|
|
t.Log("treating accountUpgradeRequired as success for PublicLink")
|
|
link, err = "bogus link to "+remote, nil
|
|
}
|
|
return
|
|
}
|
|
}
|
|
|
|
expiry := fs.Duration(120 * time.Second)
|
|
doPublicLink := wrapPublicLinkFunc(publicLinkFunc)
|
|
|
|
// if object not found
|
|
link, err := doPublicLink(ctx, file1.Path+"_does_not_exist", expiry, false)
|
|
require.Error(t, err, "Expected to get error when file doesn't exist")
|
|
require.Equal(t, "", link, "Expected link to be empty on error")
|
|
|
|
// sharing file for the first time
|
|
link1, err := doPublicLink(ctx, file1.Path, expiry, false)
|
|
require.NoError(t, err)
|
|
require.NotEqual(t, "", link1, "Link should not be empty")
|
|
|
|
link2, err := doPublicLink(ctx, file2.Path, expiry, false)
|
|
require.NoError(t, err)
|
|
require.NotEqual(t, "", link2, "Link should not be empty")
|
|
|
|
require.NotEqual(t, link1, link2, "Links to different files should differ")
|
|
|
|
// sharing file for the 2nd time
|
|
link1, err = doPublicLink(ctx, file1.Path, expiry, false)
|
|
require.NoError(t, err)
|
|
require.NotEqual(t, "", link1, "Link should not be empty")
|
|
|
|
// sharing directory for the first time
|
|
path := path.Dir(file2.Path)
|
|
link3, err := doPublicLink(ctx, path, expiry, false)
|
|
if err != nil && (errors.Is(err, fs.ErrorCantShareDirectories) || errors.Is(err, fs.ErrorObjectNotFound)) {
|
|
t.Log("skipping directory tests as not supported on this backend")
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.NotEqual(t, "", link3, "Link should not be empty")
|
|
|
|
// sharing directory for the second time
|
|
link3, err = doPublicLink(ctx, path, expiry, false)
|
|
require.NoError(t, err)
|
|
require.NotEqual(t, "", link3, "Link should not be empty")
|
|
|
|
// sharing the "root" directory in a subremote
|
|
subRemote, _, removeSubRemote, err := fstest.RandomRemote()
|
|
require.NoError(t, err)
|
|
defer removeSubRemote()
|
|
// ensure sub remote isn't empty
|
|
buf := bytes.NewBufferString("somecontent")
|
|
obji := object.NewStaticObjectInfo("somefile", time.Now(), int64(buf.Len()), true, nil, nil)
|
|
retry(t, "Put", func() error {
|
|
_, err := subRemote.Put(ctx, buf, obji)
|
|
return err
|
|
})
|
|
|
|
link4, err := wrapPublicLinkFunc(subRemote.Features().PublicLink)(ctx, "", expiry, false)
|
|
require.NoError(t, err, "Sharing root in a sub-remote should work")
|
|
require.NotEqual(t, "", link4, "Link should not be empty")
|
|
}
|
|
})
|
|
|
|
// TestSetTier tests SetTier and GetTier functionality
|
|
t.Run("SetTier", func(t *testing.T) {
|
|
skipIfNotSetTier(t)
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
setter, ok := obj.(fs.SetTierer)
|
|
assert.NotNil(t, ok)
|
|
getter, ok := obj.(fs.GetTierer)
|
|
assert.NotNil(t, ok)
|
|
// If interfaces are supported TiersToTest should contain
|
|
// at least one entry
|
|
supportedTiers := opt.TiersToTest
|
|
assert.NotEmpty(t, supportedTiers)
|
|
// test set tier changes on supported storage classes or tiers
|
|
for _, tier := range supportedTiers {
|
|
err := setter.SetTier(tier)
|
|
assert.Nil(t, err)
|
|
got := getter.GetTier()
|
|
assert.Equal(t, tier, got)
|
|
}
|
|
})
|
|
|
|
// Check to see if Fs that wrap other Objects implement all the optional methods
|
|
t.Run("ObjectCheckWrap", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
if opt.SkipObjectCheckWrap {
|
|
t.Skip("Skipping FsCheckWrap on this Fs")
|
|
}
|
|
ft := new(fs.Features).Fill(ctx, f)
|
|
if ft.UnWrap == nil {
|
|
t.Skip("Not a wrapping Fs")
|
|
}
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
_, unsupported := fs.ObjectOptionalInterfaces(obj)
|
|
for _, name := range unsupported {
|
|
if !stringsContains(name, opt.UnimplementableObjectMethods) {
|
|
t.Errorf("Missing Object wrapper for %s", name)
|
|
}
|
|
}
|
|
})
|
|
|
|
// State of remote at the moment the internal tests are called
|
|
InternalTestFiles = []fstest.Item{file1, file2}
|
|
|
|
// TestObjectRemove tests Remove
|
|
t.Run("ObjectRemove", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
// remove file1
|
|
obj := fstest.NewObject(ctx, t, f, file1.Path)
|
|
err := obj.Remove(ctx)
|
|
require.NoError(t, err)
|
|
// check listing without modtime as TestPublicLink may change the modtime
|
|
fstest.CheckListingWithPrecision(t, f, []fstest.Item{file2}, nil, fs.ModTimeNotSupported)
|
|
// Show the internal tests file2 is gone
|
|
InternalTestFiles = []fstest.Item{file2}
|
|
})
|
|
|
|
// TestAbout tests the About optional interface
|
|
t.Run("ObjectAbout", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
// Check have About
|
|
doAbout := f.Features().About
|
|
if doAbout == nil {
|
|
t.Skip("FS does not support About")
|
|
}
|
|
|
|
// Can't really check the output much!
|
|
usage, err := doAbout(context.Background())
|
|
require.NoError(t, err)
|
|
require.NotNil(t, usage)
|
|
assert.NotEqual(t, int64(0), usage.Total)
|
|
})
|
|
|
|
// Just file2 remains for Purge to clean up
|
|
|
|
// TestFsPutStream tests uploading files when size isn't known in advance.
|
|
// This may trigger large buffer allocation in some backends, keep it
|
|
// close to the end of suite. (See fs/operations/xtra_operations_test.go)
|
|
t.Run("FsPutStream", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
if f.Features().PutStream == nil {
|
|
t.Skip("FS has no PutStream interface")
|
|
}
|
|
|
|
for _, contentSize := range []int{0, 100} {
|
|
t.Run(strconv.Itoa(contentSize), func(t *testing.T) {
|
|
file := fstest.Item{
|
|
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
|
Path: "piped data.txt",
|
|
Size: -1, // use unknown size during upload
|
|
}
|
|
|
|
var (
|
|
err error
|
|
obj fs.Object
|
|
uploadHash *hash.MultiHasher
|
|
)
|
|
retry(t, "PutStream", func() error {
|
|
contents := random.String(contentSize)
|
|
buf := bytes.NewBufferString(contents)
|
|
uploadHash = hash.NewMultiHasher()
|
|
in := io.TeeReader(buf, uploadHash)
|
|
|
|
file.Size = -1
|
|
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
|
|
obj, err = f.Features().PutStream(ctx, in, obji)
|
|
return err
|
|
})
|
|
file.Hashes = uploadHash.Sums()
|
|
file.Size = int64(contentSize) // use correct size when checking
|
|
file.Check(t, obj, f.Precision())
|
|
// Re-read the object and check again
|
|
obj = fstest.NewObject(ctx, t, f, file.Path)
|
|
file.Check(t, obj, f.Precision())
|
|
require.NoError(t, obj.Remove(ctx))
|
|
})
|
|
}
|
|
})
|
|
|
|
// TestInternal calls InternalTest() on the Fs
|
|
t.Run("Internal", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
if it, ok := f.(InternalTester); ok {
|
|
it.InternalTest(t)
|
|
} else {
|
|
t.Skipf("%T does not implement InternalTester", f)
|
|
}
|
|
})
|
|
|
|
})
|
|
|
|
// TestFsPutChunked may trigger large buffer allocation with
|
|
// some backends (see fs/operations/xtra_operations_test.go),
|
|
// keep it closer to the end of suite.
|
|
t.Run("FsPutChunked", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
if testing.Short() {
|
|
t.Skip("not running with -short")
|
|
}
|
|
|
|
if opt.ChunkedUpload.Skip {
|
|
t.Skip("skipping as ChunkedUpload.Skip is set")
|
|
}
|
|
|
|
setUploadChunkSizer, _ := f.(SetUploadChunkSizer)
|
|
if setUploadChunkSizer == nil {
|
|
t.Skipf("%T does not implement SetUploadChunkSizer", f)
|
|
}
|
|
|
|
setUploadCutoffer, _ := f.(SetUploadCutoffer)
|
|
|
|
minChunkSize := opt.ChunkedUpload.MinChunkSize
|
|
if minChunkSize < 100 {
|
|
minChunkSize = 100
|
|
}
|
|
if opt.ChunkedUpload.CeilChunkSize != nil {
|
|
minChunkSize = opt.ChunkedUpload.CeilChunkSize(minChunkSize)
|
|
}
|
|
|
|
maxChunkSize := 2 * fs.Mebi
|
|
if maxChunkSize < 2*minChunkSize {
|
|
maxChunkSize = 2 * minChunkSize
|
|
}
|
|
if opt.ChunkedUpload.MaxChunkSize > 0 && maxChunkSize > opt.ChunkedUpload.MaxChunkSize {
|
|
maxChunkSize = opt.ChunkedUpload.MaxChunkSize
|
|
}
|
|
if opt.ChunkedUpload.CeilChunkSize != nil {
|
|
maxChunkSize = opt.ChunkedUpload.CeilChunkSize(maxChunkSize)
|
|
}
|
|
|
|
next := func(f func(fs.SizeSuffix) fs.SizeSuffix) fs.SizeSuffix {
|
|
s := f(minChunkSize)
|
|
if s > maxChunkSize {
|
|
s = minChunkSize
|
|
}
|
|
return s
|
|
}
|
|
|
|
chunkSizes := fs.SizeSuffixList{
|
|
minChunkSize,
|
|
minChunkSize + (maxChunkSize-minChunkSize)/3,
|
|
next(NextPowerOfTwo),
|
|
next(NextMultipleOf(100000)),
|
|
next(NextMultipleOf(100001)),
|
|
maxChunkSize,
|
|
}
|
|
chunkSizes.Sort()
|
|
|
|
// Set the minimum chunk size, upload cutoff and reset it at the end
|
|
oldChunkSize, err := setUploadChunkSizer.SetUploadChunkSize(minChunkSize)
|
|
require.NoError(t, err)
|
|
var oldUploadCutoff fs.SizeSuffix
|
|
if setUploadCutoffer != nil {
|
|
oldUploadCutoff, err = setUploadCutoffer.SetUploadCutoff(minChunkSize)
|
|
require.NoError(t, err)
|
|
}
|
|
defer func() {
|
|
_, err := setUploadChunkSizer.SetUploadChunkSize(oldChunkSize)
|
|
assert.NoError(t, err)
|
|
if setUploadCutoffer != nil {
|
|
_, err := setUploadCutoffer.SetUploadCutoff(oldUploadCutoff)
|
|
assert.NoError(t, err)
|
|
}
|
|
}()
|
|
|
|
var lastCs fs.SizeSuffix
|
|
for _, cs := range chunkSizes {
|
|
if cs <= lastCs {
|
|
continue
|
|
}
|
|
if opt.ChunkedUpload.CeilChunkSize != nil {
|
|
cs = opt.ChunkedUpload.CeilChunkSize(cs)
|
|
}
|
|
lastCs = cs
|
|
|
|
t.Run(cs.String(), func(t *testing.T) {
|
|
_, err := setUploadChunkSizer.SetUploadChunkSize(cs)
|
|
require.NoError(t, err)
|
|
if setUploadCutoffer != nil {
|
|
_, err = setUploadCutoffer.SetUploadCutoff(cs)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
var testChunks []fs.SizeSuffix
|
|
if opt.ChunkedUpload.NeedMultipleChunks {
|
|
// If NeedMultipleChunks is set then test with > cs
|
|
testChunks = []fs.SizeSuffix{cs + 1, 2 * cs, 2*cs + 1}
|
|
} else {
|
|
testChunks = []fs.SizeSuffix{cs - 1, cs, 2*cs + 1}
|
|
}
|
|
|
|
for _, fileSize := range testChunks {
|
|
t.Run(fmt.Sprintf("%d", fileSize), func(t *testing.T) {
|
|
TestPutLarge(ctx, t, f, &fstest.Item{
|
|
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
|
Path: fmt.Sprintf("chunked-%s-%s.bin", cs.String(), fileSize.String()),
|
|
Size: int64(fileSize),
|
|
})
|
|
t.Run("Streamed", func(t *testing.T) {
|
|
if f.Features().PutStream == nil {
|
|
t.Skip("FS has no PutStream interface")
|
|
}
|
|
TestPutLargeStreamed(ctx, t, f, &fstest.Item{
|
|
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
|
Path: fmt.Sprintf("chunked-%s-%s-streamed.bin", cs.String(), fileSize.String()),
|
|
Size: int64(fileSize),
|
|
})
|
|
})
|
|
})
|
|
}
|
|
})
|
|
}
|
|
})
|
|
|
|
// Copy files with chunked copy if available
|
|
t.Run("FsCopyChunked", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
if testing.Short() {
|
|
t.Skip("not running with -short")
|
|
}
|
|
|
|
// Check have Copy
|
|
doCopy := f.Features().Copy
|
|
if doCopy == nil {
|
|
t.Skip("FS has no Copier interface")
|
|
}
|
|
|
|
if opt.ChunkedUpload.Skip {
|
|
t.Skip("skipping as ChunkedUpload.Skip is set")
|
|
}
|
|
|
|
if strings.HasPrefix(f.Name(), "serves3") || strings.HasPrefix(f.Name(), "TestS3Rclone") {
|
|
t.Skip("FIXME skip test - see #7454")
|
|
}
|
|
|
|
do, _ := f.(SetCopyCutoffer)
|
|
if do == nil {
|
|
t.Skipf("%T does not implement SetCopyCutoff", f)
|
|
}
|
|
|
|
minChunkSize := opt.ChunkedUpload.MinChunkSize
|
|
if minChunkSize < 100 {
|
|
minChunkSize = 100
|
|
}
|
|
if opt.ChunkedUpload.CeilChunkSize != nil {
|
|
minChunkSize = opt.ChunkedUpload.CeilChunkSize(minChunkSize)
|
|
}
|
|
|
|
chunkSizes := fs.SizeSuffixList{
|
|
minChunkSize,
|
|
minChunkSize + 1,
|
|
2*minChunkSize - 1,
|
|
2 * minChunkSize,
|
|
2*minChunkSize + 1,
|
|
}
|
|
for _, chunkSize := range chunkSizes {
|
|
t.Run(fmt.Sprintf("%d", chunkSize), func(t *testing.T) {
|
|
contents := random.String(int(chunkSize))
|
|
item := fstest.NewItem("chunked-copy", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
src := PutTestContents(ctx, t, f, &item, contents, true)
|
|
defer func() {
|
|
assert.NoError(t, src.Remove(ctx))
|
|
}()
|
|
|
|
var itemCopy = item
|
|
itemCopy.Path += ".copy"
|
|
|
|
// Set copy cutoff to mininum value so we make chunks
|
|
origCutoff, err := do.SetCopyCutoff(minChunkSize)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
_, err = do.SetCopyCutoff(origCutoff)
|
|
require.NoError(t, err)
|
|
}()
|
|
|
|
// Do the copy
|
|
dst, err := doCopy(ctx, src, itemCopy.Path)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
assert.NoError(t, dst.Remove(ctx))
|
|
}()
|
|
|
|
// Check size
|
|
assert.Equal(t, src.Size(), dst.Size())
|
|
|
|
// Check modtime
|
|
srcModTime := src.ModTime(ctx)
|
|
dstModTime := dst.ModTime(ctx)
|
|
assert.True(t, srcModTime.Equal(dstModTime))
|
|
|
|
// Make sure contents are correct
|
|
gotContents := ReadObject(ctx, t, dst, -1)
|
|
assert.Equal(t, contents, gotContents)
|
|
})
|
|
}
|
|
})
|
|
|
|
// TestFsUploadUnknownSize ensures Fs.Put() and Object.Update() don't panic when
|
|
// src.Size() == -1
|
|
//
|
|
// This may trigger large buffer allocation in some backends, keep it
|
|
// closer to the suite end. (See fs/operations/xtra_operations_test.go)
|
|
t.Run("FsUploadUnknownSize", func(t *testing.T) {
|
|
skipIfNotOk(t)
|
|
|
|
t.Run("FsPutUnknownSize", func(t *testing.T) {
|
|
defer func() {
|
|
assert.Nil(t, recover(), "Fs.Put() should not panic when src.Size() == -1")
|
|
}()
|
|
|
|
contents := random.String(100)
|
|
in := bytes.NewBufferString(contents)
|
|
|
|
obji := object.NewStaticObjectInfo("unknown-size-put.txt", fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil)
|
|
obj, err := f.Put(ctx, in, obji)
|
|
if err == nil {
|
|
require.NoError(t, obj.Remove(ctx), "successfully uploaded unknown-sized file but failed to remove")
|
|
}
|
|
// if err != nil: it's okay as long as no panic
|
|
})
|
|
|
|
t.Run("FsUpdateUnknownSize", func(t *testing.T) {
|
|
unknownSizeUpdateFile := fstest.Item{
|
|
ModTime: fstest.Time("2002-02-03T04:05:06.499999999Z"),
|
|
Path: "unknown-size-update.txt",
|
|
}
|
|
|
|
testPut(ctx, t, f, &unknownSizeUpdateFile)
|
|
|
|
defer func() {
|
|
assert.Nil(t, recover(), "Object.Update() should not panic when src.Size() == -1")
|
|
}()
|
|
|
|
newContents := random.String(200)
|
|
in := bytes.NewBufferString(newContents)
|
|
|
|
obj := fstest.NewObject(ctx, t, f, unknownSizeUpdateFile.Path)
|
|
obji := object.NewStaticObjectInfo(unknownSizeUpdateFile.Path, unknownSizeUpdateFile.ModTime, -1, true, nil, obj.Fs())
|
|
err := obj.Update(ctx, in, obji)
|
|
if err == nil {
|
|
require.NoError(t, obj.Remove(ctx), "successfully updated object with unknown-sized source but failed to remove")
|
|
}
|
|
// if err != nil: it's okay as long as no panic
|
|
})
|
|
|
|
})
|
|
|
|
// TestFsRootCollapse tests if the root of an fs "collapses" to the
|
|
// absolute root. It creates a new fs of the same backend type with its
|
|
// root set to a *nonexistent* folder, and attempts to read the info of
|
|
// an object in that folder, whose name is taken from a directory that
|
|
// exists in the absolute root.
|
|
// This test is added after
|
|
// https://github.com/rclone/rclone/issues/3164.
|
|
t.Run("FsRootCollapse", func(t *testing.T) {
|
|
deepRemoteName := subRemoteName + "/deeper/nonexisting/directory"
|
|
deepRemote, err := fs.NewFs(context.Background(), deepRemoteName)
|
|
require.NoError(t, err)
|
|
|
|
colonIndex := strings.IndexRune(deepRemoteName, ':')
|
|
firstSlashIndex := strings.IndexRune(deepRemoteName, '/')
|
|
firstDir := deepRemoteName[colonIndex+1 : firstSlashIndex]
|
|
_, err = deepRemote.NewObject(ctx, firstDir)
|
|
require.Equal(t, fs.ErrorObjectNotFound, err)
|
|
// If err is not fs.ErrorObjectNotFound, it means the backend is
|
|
// somehow confused about root and absolute root.
|
|
})
|
|
|
|
// FsDirSetModTime tests setting the mod time on a directory if possible
|
|
t.Run("FsDirSetModTime", func(t *testing.T) {
|
|
const name = "dir-mod-time"
|
|
do := f.Features().DirSetModTime
|
|
if do == nil {
|
|
t.Skip("FS has no DirSetModTime interface")
|
|
}
|
|
|
|
// Set ModTime on non existing directory should return error
|
|
t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
|
|
err := do(ctx, name, t1)
|
|
require.Error(t, err)
|
|
|
|
// Make the directory and try again
|
|
err = f.Mkdir(ctx, name)
|
|
require.NoError(t, err)
|
|
err = do(ctx, name, t1)
|
|
require.NoError(t, err)
|
|
|
|
// Check the modtime got set properly
|
|
dir := fstest.NewDirectory(ctx, t, f, name)
|
|
fstest.CheckDirModTime(ctx, t, f, dir, t1)
|
|
|
|
// Tidy up
|
|
err = f.Rmdir(ctx, name)
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
var testMetadata = fs.Metadata{
|
|
// System metadata supported by all backends
|
|
"mtime": "2001-02-03T04:05:06.499999999Z",
|
|
// User metadata
|
|
"potato": "jersey",
|
|
}
|
|
var testMetadata2 = fs.Metadata{
|
|
// System metadata supported by all backends
|
|
"mtime": "2002-02-03T04:05:06.499999999Z",
|
|
// User metadata
|
|
"potato": "king edwards",
|
|
}
|
|
|
|
// FsMkdirMetadata tests creating a directory with metadata if possible
|
|
t.Run("FsMkdirMetadata", func(t *testing.T) {
|
|
ctx, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
const name = "dir-metadata"
|
|
do := f.Features().MkdirMetadata
|
|
if do == nil {
|
|
t.Skip("FS has no MkdirMetadata interface")
|
|
}
|
|
assert.True(t, f.Features().WriteDirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata")
|
|
|
|
// Create the directory from fresh
|
|
dir, err := do(ctx, name, testMetadata)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, dir)
|
|
|
|
// Check the returned directory and one read from the listing
|
|
fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata)
|
|
fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata)
|
|
|
|
// Now update the metadata on the existing directory
|
|
t.Run("Update", func(t *testing.T) {
|
|
dir, err := do(ctx, name, testMetadata2)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, dir)
|
|
|
|
// Check the returned directory and one read from the listing
|
|
fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata2)
|
|
// The TestUnionPolicy2 has randomness in it so it sets metadata on
|
|
// one directory but can read a different one from the listing.
|
|
if f.Name() != "TestUnionPolicy2" {
|
|
fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata2)
|
|
}
|
|
})
|
|
|
|
// Now test the Directory methods
|
|
t.Run("CheckDirectory", func(t *testing.T) {
|
|
_, ok := dir.(fs.Object)
|
|
assert.False(t, ok, "Directory must not type assert to Object")
|
|
_, ok = dir.(fs.ObjectInfo)
|
|
assert.False(t, ok, "Directory must not type assert to ObjectInfo")
|
|
})
|
|
|
|
// Tidy up
|
|
err = f.Rmdir(ctx, name)
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
// FsDirectory checks methods on the directory object
|
|
t.Run("FsDirectory", func(t *testing.T) {
|
|
ctx, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
const name = "dir-methods"
|
|
features := f.Features()
|
|
|
|
if !features.CanHaveEmptyDirectories {
|
|
t.Skip("Can't test if can't have empty directories")
|
|
}
|
|
if !features.ReadDirMetadata &&
|
|
!features.WriteDirMetadata &&
|
|
!features.WriteDirSetModTime &&
|
|
!features.UserDirMetadata &&
|
|
!features.Overlay &&
|
|
features.UnWrap == nil {
|
|
t.Skip("FS has no Directory methods and doesn't Wrap")
|
|
}
|
|
|
|
// Create a directory to start with
|
|
err := f.Mkdir(ctx, name)
|
|
require.NoError(t, err)
|
|
|
|
// Get the directory object
|
|
dir := fstest.NewDirectory(ctx, t, f, name)
|
|
_, ok := dir.(fs.Object)
|
|
assert.False(t, ok, "Directory must not type assert to Object")
|
|
_, ok = dir.(fs.ObjectInfo)
|
|
assert.False(t, ok, "Directory must not type assert to ObjectInfo")
|
|
|
|
// Now test the directory methods
|
|
t.Run("ReadDirMetadata", func(t *testing.T) {
|
|
if !features.ReadDirMetadata {
|
|
t.Skip("Directories don't support ReadDirMetadata")
|
|
}
|
|
if f.Name() == "TestUnionPolicy3" {
|
|
t.Skipf("Test unreliable on %q", f.Name())
|
|
}
|
|
fstest.CheckEntryMetadata(ctx, t, f, dir, fs.Metadata{
|
|
"mtime": dir.ModTime(ctx).Format(time.RFC3339Nano),
|
|
})
|
|
})
|
|
|
|
t.Run("WriteDirMetadata", func(t *testing.T) {
|
|
if !features.WriteDirMetadata {
|
|
t.Skip("Directories don't support WriteDirMetadata")
|
|
}
|
|
assert.NotNil(t, features.MkdirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata")
|
|
do, ok := dir.(fs.SetMetadataer)
|
|
require.True(t, ok, "Expected to find SetMetadata method on Directory")
|
|
err := do.SetMetadata(ctx, testMetadata)
|
|
require.NoError(t, err)
|
|
|
|
fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata)
|
|
fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata)
|
|
})
|
|
|
|
t.Run("WriteDirSetModTime", func(t *testing.T) {
|
|
if !features.WriteDirSetModTime {
|
|
t.Skip("Directories don't support WriteDirSetModTime")
|
|
}
|
|
assert.NotNil(t, features.DirSetModTime, "Backends must support Directory.SetModTime and Fs.DirSetModTime")
|
|
|
|
t1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
|
|
|
|
do, ok := dir.(fs.SetModTimer)
|
|
require.True(t, ok, "Expected to find SetMetadata method on Directory")
|
|
err := do.SetModTime(ctx, t1)
|
|
require.NoError(t, err)
|
|
|
|
fstest.CheckDirModTime(ctx, t, f, dir, t1)
|
|
fstest.CheckDirModTime(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), t1)
|
|
})
|
|
|
|
// Check to see if Fs that wrap other Directories implement all the optional methods
|
|
t.Run("DirectoryCheckWrap", func(t *testing.T) {
|
|
if opt.SkipDirectoryCheckWrap {
|
|
t.Skip("Skipping DirectoryCheckWrap on this Fs")
|
|
}
|
|
if !features.Overlay && features.UnWrap == nil {
|
|
t.Skip("Not a wrapping Fs")
|
|
}
|
|
_, unsupported := fs.DirectoryOptionalInterfaces(dir)
|
|
for _, name := range unsupported {
|
|
if !stringsContains(name, opt.UnimplementableDirectoryMethods) {
|
|
t.Errorf("Missing Directory wrapper for %s", name)
|
|
}
|
|
}
|
|
})
|
|
|
|
// Tidy up
|
|
err = f.Rmdir(ctx, name)
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
// Purge the folder
|
|
err = operations.Purge(ctx, f, "")
|
|
if !errors.Is(err, fs.ErrorDirNotFound) {
|
|
require.NoError(t, err)
|
|
}
|
|
purged = true
|
|
fstest.CheckListing(t, f, []fstest.Item{})
|
|
|
|
// Check purging again if not bucket-based
|
|
if !isBucketBasedButNotRoot(f) {
|
|
err = operations.Purge(ctx, f, "")
|
|
assert.Error(t, err, "Expecting error after on second purge")
|
|
if !errors.Is(err, fs.ErrorDirNotFound) {
|
|
t.Log("Warning: this should produce fs.ErrorDirNotFound")
|
|
}
|
|
}
|
|
|
|
})
|
|
|
|
// Check directory is purged
|
|
if !purged {
|
|
_ = operations.Purge(ctx, f, "")
|
|
}
|
|
|
|
t.Run("FsShutdown", func(t *testing.T) {
|
|
do := f.Features().Shutdown
|
|
if do == nil {
|
|
t.Skip("Shutdown method not supported")
|
|
}
|
|
require.NoError(t, do(ctx))
|
|
require.NoError(t, do(ctx), "must be able to call Shutdown twice")
|
|
})
|
|
|
|
// Remove the local directory so we don't clutter up /tmp
|
|
if strings.HasPrefix(remoteName, "/") {
|
|
t.Log("remoteName", remoteName)
|
|
// Remove temp directory
|
|
err := os.Remove(remoteName)
|
|
require.NoError(t, err)
|
|
}
|
|
}
|