mirror of
https://github.com/rclone/rclone.git
synced 2025-02-17 19:11:33 +01:00
Implement server side copies if possible - fixes #99
Add optional fs.Copier interface Implemented for * swift * s3 * drive * dropbox * google cloud storage
This commit is contained in:
parent
fedf81c2b7
commit
a96b522958
2
Makefile
2
Makefile
@ -66,4 +66,4 @@ retag:
|
|||||||
git tag -f $(LAST_TAG)
|
git tag -f $(LAST_TAG)
|
||||||
|
|
||||||
gen_tests:
|
gen_tests:
|
||||||
cd fstest/fstests && go run gen_tests.go
|
cd fstest/fstests && go generate
|
||||||
|
@ -103,6 +103,37 @@ Enter an interactive configuration session.
|
|||||||
|
|
||||||
Prints help on rclone commands and options.
|
Prints help on rclone commands and options.
|
||||||
|
|
||||||
|
Server Side Copy
|
||||||
|
----------------
|
||||||
|
|
||||||
|
Drive, S3, Dropbox, Swift and Google Cloud Storage support server side
|
||||||
|
copy.
|
||||||
|
|
||||||
|
This means if you want to copy one folder to another then rclone won't
|
||||||
|
download all the files and re-upload them; it will instruct the server
|
||||||
|
to copy them in place.
|
||||||
|
|
||||||
|
Eg
|
||||||
|
|
||||||
|
rclone copy s3:oldbucket s3:newbucket
|
||||||
|
|
||||||
|
Will copy the contents of `oldbucket` to `newbucket` without
|
||||||
|
downloading and re-uploading.
|
||||||
|
|
||||||
|
Remotes which don't support server side copy (eg local) **will**
|
||||||
|
download and re-upload in this case.
|
||||||
|
|
||||||
|
Server side copies are used with `sync` and `copy` and will be
|
||||||
|
identified in the log when using the `-v` flag.
|
||||||
|
|
||||||
|
Server side copies will only be attempted if the remote names are the
|
||||||
|
same.
|
||||||
|
|
||||||
|
This can be used when scripting to make aged backups efficiently, eg
|
||||||
|
|
||||||
|
rclone sync remote:current-backup remote:previous-backup
|
||||||
|
rclone sync /path/to/files remote:current-backup
|
||||||
|
|
||||||
Options
|
Options
|
||||||
-------
|
-------
|
||||||
|
|
||||||
|
@ -734,6 +734,35 @@ func (f *FsDrive) ListDir() fs.DirChan {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Creates a drive.File info from the parameters passed in and a half
|
||||||
|
// finished FsObjectDrive which must have setMetaData called on it
|
||||||
|
//
|
||||||
|
// Used to create new objects
|
||||||
|
func (f *FsDrive) createFileInfo(remote string, modTime time.Time, size int64) (*FsObjectDrive, *drive.File, error) {
|
||||||
|
// Temporary FsObject under construction
|
||||||
|
o := &FsObjectDrive{
|
||||||
|
drive: f,
|
||||||
|
remote: remote,
|
||||||
|
bytes: size,
|
||||||
|
}
|
||||||
|
|
||||||
|
directory, leaf := splitPath(remote)
|
||||||
|
directoryId, err := f.findDir(directory, true)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("Couldn't find or make directory: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Define the metadata for the file we are going to create.
|
||||||
|
createInfo := &drive.File{
|
||||||
|
Title: leaf,
|
||||||
|
Description: leaf,
|
||||||
|
Parents: []*drive.ParentReference{{Id: directoryId}},
|
||||||
|
MimeType: fs.MimeType(o),
|
||||||
|
ModifiedDate: modTime.Format(timeFormatOut),
|
||||||
|
}
|
||||||
|
return o, createInfo, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Put the object
|
// Put the object
|
||||||
//
|
//
|
||||||
// This assumes that the object doesn't not already exists - if you
|
// This assumes that the object doesn't not already exists - if you
|
||||||
@ -744,22 +773,9 @@ func (f *FsDrive) ListDir() fs.DirChan {
|
|||||||
//
|
//
|
||||||
// The new object may have been created if an error is returned
|
// The new object may have been created if an error is returned
|
||||||
func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
func (f *FsDrive) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
|
||||||
// Temporary FsObject under construction
|
o, createInfo, err := f.createFileInfo(remote, modTime, size)
|
||||||
o := &FsObjectDrive{drive: f, remote: remote}
|
|
||||||
|
|
||||||
directory, leaf := splitPath(o.remote)
|
|
||||||
directoryId, err := f.findDir(directory, true)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return o, fmt.Errorf("Couldn't find or make directory: %s", err)
|
return nil, err
|
||||||
}
|
|
||||||
|
|
||||||
// Define the metadata for the file we are going to create.
|
|
||||||
createInfo := &drive.File{
|
|
||||||
Title: leaf,
|
|
||||||
Description: leaf,
|
|
||||||
Parents: []*drive.ParentReference{{Id: directoryId}},
|
|
||||||
MimeType: fs.MimeType(o),
|
|
||||||
ModifiedDate: modTime.Format(timeFormatOut),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var info *drive.File
|
var info *drive.File
|
||||||
@ -830,6 +846,39 @@ func (fs *FsDrive) Precision() time.Duration {
|
|||||||
return time.Millisecond
|
return time.Millisecond
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Copy src to this remote using server side copy operations.
|
||||||
|
//
|
||||||
|
// This is stored with the remote path given
|
||||||
|
//
|
||||||
|
// It returns the destination Object and a possible error
|
||||||
|
//
|
||||||
|
// Will only be called if src.Fs().Name() == f.Name()
|
||||||
|
//
|
||||||
|
// If it isn't possible then return fs.ErrorCantCopy
|
||||||
|
func (f *FsDrive) Copy(src fs.Object, remote string) (fs.Object, error) {
|
||||||
|
srcObj, ok := src.(*FsObjectDrive)
|
||||||
|
if !ok {
|
||||||
|
fs.Debug(src, "Can't copy - not same remote type")
|
||||||
|
return nil, fs.ErrorCantCopy
|
||||||
|
}
|
||||||
|
|
||||||
|
o, createInfo, err := f.createFileInfo(remote, srcObj.ModTime(), srcObj.bytes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var info *drive.File
|
||||||
|
o.drive.call(&err, func() {
|
||||||
|
info, err = o.drive.svc.Files.Copy(srcObj.id, createInfo).Do()
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
o.setMetaData(info)
|
||||||
|
return o, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Purge deletes all the files and the container
|
// Purge deletes all the files and the container
|
||||||
//
|
//
|
||||||
// Optional interface: Only implement this if you have a way of
|
// Optional interface: Only implement this if you have a way of
|
||||||
@ -1051,4 +1100,5 @@ func (o *FsObjectDrive) Remove() error {
|
|||||||
// Check the interfaces are satisfied
|
// Check the interfaces are satisfied
|
||||||
var _ fs.Fs = &FsDrive{}
|
var _ fs.Fs = &FsDrive{}
|
||||||
var _ fs.Purger = &FsDrive{}
|
var _ fs.Purger = &FsDrive{}
|
||||||
|
var _ fs.Copier = &FsDrive{}
|
||||||
var _ fs.Object = &FsObjectDrive{}
|
var _ fs.Object = &FsObjectDrive{}
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
// Test Drive filesystem interface
|
// Test Drive filesystem interface
|
||||||
//
|
//
|
||||||
// Automatically generated - DO NOT EDIT
|
// Automatically generated - DO NOT EDIT
|
||||||
// Regenerate with: go run gen_tests.go or make gen_tests
|
// Regenerate with: make gen_tests
|
||||||
package drive_test
|
package drive_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
|
|||||||
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
||||||
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
||||||
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
||||||
|
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
|
||||||
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
||||||
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
||||||
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
||||||
|
@ -414,6 +414,35 @@ func (f *FsDropbox) Precision() time.Duration {
|
|||||||
return fs.ModTimeNotSupported
|
return fs.ModTimeNotSupported
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Copy src to this remote using server side copy operations.
|
||||||
|
//
|
||||||
|
// This is stored with the remote path given
|
||||||
|
//
|
||||||
|
// It returns the destination Object and a possible error
|
||||||
|
//
|
||||||
|
// Will only be called if src.Fs().Name() == f.Name()
|
||||||
|
//
|
||||||
|
// If it isn't possible then return fs.ErrorCantCopy
|
||||||
|
func (f *FsDropbox) Copy(src fs.Object, remote string) (fs.Object, error) {
|
||||||
|
srcObj, ok := src.(*FsObjectDropbox)
|
||||||
|
if !ok {
|
||||||
|
fs.Debug(src, "Can't copy - not same remote type")
|
||||||
|
return nil, fs.ErrorCantCopy
|
||||||
|
}
|
||||||
|
|
||||||
|
// Temporary FsObject under construction
|
||||||
|
dstObj := &FsObjectDropbox{dropbox: f, remote: remote}
|
||||||
|
|
||||||
|
srcPath := srcObj.remotePath()
|
||||||
|
dstPath := dstObj.remotePath()
|
||||||
|
entry, err := f.db.Copy(srcPath, dstPath, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("Copy failed: %s", err)
|
||||||
|
}
|
||||||
|
dstObj.setMetadataFromEntry(entry)
|
||||||
|
return dstObj, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Purge deletes all the files and the container
|
// Purge deletes all the files and the container
|
||||||
//
|
//
|
||||||
// Optional interface: Only implement this if you have a way of
|
// Optional interface: Only implement this if you have a way of
|
||||||
@ -575,5 +604,6 @@ func (o *FsObjectDropbox) Remove() error {
|
|||||||
|
|
||||||
// Check the interfaces are satisfied
|
// Check the interfaces are satisfied
|
||||||
var _ fs.Fs = &FsDropbox{}
|
var _ fs.Fs = &FsDropbox{}
|
||||||
|
var _ fs.Copier = &FsDropbox{}
|
||||||
var _ fs.Purger = &FsDropbox{}
|
var _ fs.Purger = &FsDropbox{}
|
||||||
var _ fs.Object = &FsObjectDropbox{}
|
var _ fs.Object = &FsObjectDropbox{}
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
// Test Dropbox filesystem interface
|
// Test Dropbox filesystem interface
|
||||||
//
|
//
|
||||||
// Automatically generated - DO NOT EDIT
|
// Automatically generated - DO NOT EDIT
|
||||||
// Regenerate with: go run gen_tests.go or make gen_tests
|
// Regenerate with: make gen_tests
|
||||||
package dropbox_test
|
package dropbox_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
|
|||||||
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
||||||
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
||||||
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
||||||
|
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
|
||||||
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
||||||
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
||||||
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
||||||
|
14
fs/fs.go
14
fs/fs.go
@ -24,6 +24,7 @@ var (
|
|||||||
fsRegistry []*FsInfo
|
fsRegistry []*FsInfo
|
||||||
// Error returned by NewFs if not found in config file
|
// Error returned by NewFs if not found in config file
|
||||||
NotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
|
NotFoundInConfigFile = fmt.Errorf("Didn't find section in config file")
|
||||||
|
ErrorCantCopy = fmt.Errorf("Can't copy object - incompatible remotes")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Filesystem info
|
// Filesystem info
|
||||||
@ -149,6 +150,19 @@ type Purger interface {
|
|||||||
Purge() error
|
Purge() error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type Copier interface {
|
||||||
|
// Copy src to this remote using server side copy operations.
|
||||||
|
//
|
||||||
|
// This is stored with the remote path given
|
||||||
|
//
|
||||||
|
// It returns the destination Object and a possible error
|
||||||
|
//
|
||||||
|
// Will only be called if src.Fs().Name() == f.Name()
|
||||||
|
//
|
||||||
|
// If it isn't possible then return fs.ErrorCantCopy
|
||||||
|
Copy(src Object, remote string) (Object, error)
|
||||||
|
}
|
||||||
|
|
||||||
// An optional interface for error as to whether the operation should be retried
|
// An optional interface for error as to whether the operation should be retried
|
||||||
//
|
//
|
||||||
// This should be returned from Update or Put methods as required
|
// This should be returned from Update or Put methods as required
|
||||||
|
@ -89,5 +89,23 @@ func (f *Limited) Precision() time.Duration {
|
|||||||
return f.fs.Precision()
|
return f.fs.Precision()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Copy src to this remote using server side copy operations.
|
||||||
|
//
|
||||||
|
// This is stored with the remote path given
|
||||||
|
//
|
||||||
|
// It returns the destination Object and a possible error
|
||||||
|
//
|
||||||
|
// Will only be called if src.Fs().Name() == f.Name()
|
||||||
|
//
|
||||||
|
// If it isn't possible then return fs.ErrorCantCopy
|
||||||
|
func (f *Limited) Copy(src Object, remote string) (Object, error) {
|
||||||
|
fCopy, ok := f.fs.(Copier)
|
||||||
|
if !ok {
|
||||||
|
return nil, ErrorCantCopy
|
||||||
|
}
|
||||||
|
return fCopy.Copy(src, remote)
|
||||||
|
}
|
||||||
|
|
||||||
// Check the interfaces are satisfied
|
// Check the interfaces are satisfied
|
||||||
var _ Fs = &Limited{}
|
var _ Fs = &Limited{}
|
||||||
|
var _ Copier = &Limited{}
|
||||||
|
@ -173,24 +173,40 @@ func Copy(f Fs, dst, src Object) {
|
|||||||
const maxTries = 10
|
const maxTries = 10
|
||||||
tries := 0
|
tries := 0
|
||||||
doUpdate := dst != nil
|
doUpdate := dst != nil
|
||||||
|
var err, inErr error
|
||||||
tryAgain:
|
tryAgain:
|
||||||
in0, err := src.Open()
|
// Try server side copy first - if has optional interface and
|
||||||
if err != nil {
|
// is same underlying remote
|
||||||
Stats.Error()
|
actionTaken := "Copied (server side copy)"
|
||||||
ErrorLog(src, "Failed to open: %s", err)
|
if fCopy, ok := f.(Copier); ok && src.Fs().Name() == f.Name() {
|
||||||
return
|
var newDst Object
|
||||||
}
|
newDst, err = fCopy.Copy(src, src.Remote())
|
||||||
in := NewAccount(in0) // account the transfer
|
if err == nil {
|
||||||
|
dst = newDst
|
||||||
var actionTaken string
|
}
|
||||||
if doUpdate {
|
|
||||||
actionTaken = "Copied (updated existing)"
|
|
||||||
err = dst.Update(in, src.ModTime(), src.Size())
|
|
||||||
} else {
|
} else {
|
||||||
actionTaken = "Copied (new)"
|
err = ErrorCantCopy
|
||||||
dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size())
|
}
|
||||||
|
// If can't server side copy, do it manually
|
||||||
|
if err == ErrorCantCopy {
|
||||||
|
var in0 io.ReadCloser
|
||||||
|
in0, err = src.Open()
|
||||||
|
if err != nil {
|
||||||
|
Stats.Error()
|
||||||
|
ErrorLog(src, "Failed to open: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
in := NewAccount(in0) // account the transfer
|
||||||
|
|
||||||
|
if doUpdate {
|
||||||
|
actionTaken = "Copied (updated existing)"
|
||||||
|
err = dst.Update(in, src.ModTime(), src.Size())
|
||||||
|
} else {
|
||||||
|
actionTaken = "Copied (new)"
|
||||||
|
dst, err = f.Put(in, src.Remote(), src.ModTime(), src.Size())
|
||||||
|
}
|
||||||
|
inErr = in.Close()
|
||||||
}
|
}
|
||||||
inErr := in.Close()
|
|
||||||
// Retry if err returned a retry error
|
// Retry if err returned a retry error
|
||||||
if r, ok := err.(Retry); ok && r.Retry() && tries < maxTries {
|
if r, ok := err.(Retry); ok && r.Retry() && tries < maxTries {
|
||||||
tries++
|
tries++
|
||||||
@ -279,7 +295,7 @@ func PairChecker(in ObjectPairChan, out ObjectPairChan, wg *sync.WaitGroup) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Read Objects on in and copy them
|
// Read Objects on in and copy them
|
||||||
func Copier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
|
func PairCopier(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
for pair := range in {
|
for pair := range in {
|
||||||
src := pair.src
|
src := pair.src
|
||||||
@ -364,7 +380,7 @@ func Sync(fdst, fsrc Fs, Delete bool) error {
|
|||||||
var copierWg sync.WaitGroup
|
var copierWg sync.WaitGroup
|
||||||
copierWg.Add(Config.Transfers)
|
copierWg.Add(Config.Transfers)
|
||||||
for i := 0; i < Config.Transfers; i++ {
|
for i := 0; i < Config.Transfers; i++ {
|
||||||
go Copier(to_be_uploaded, fdst, &copierWg)
|
go PairCopier(to_be_uploaded, fdst, &copierWg)
|
||||||
}
|
}
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
|
@ -127,6 +127,28 @@ func TestCopy(t *testing.T) {
|
|||||||
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
|
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Test a server side copy if possible, or the backup path if not
|
||||||
|
func TestServerSideCopy(t *testing.T) {
|
||||||
|
fremoteCopy, finaliseCopy, err := fstest.RandomRemote(*RemoteName, *SubDir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to open remote copy %q: %v", *RemoteName, err)
|
||||||
|
}
|
||||||
|
defer finaliseCopy()
|
||||||
|
t.Logf("Server side copy (if possible) %v -> %v", fremote, fremoteCopy)
|
||||||
|
|
||||||
|
err = fs.Sync(fremoteCopy, fremote, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Server Side Copy failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
items := []fstest.Item{
|
||||||
|
{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
|
||||||
|
}
|
||||||
|
|
||||||
|
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
|
||||||
|
fstest.CheckListingWithPrecision(t, fremoteCopy, items, fs.Config.ModifyWindow)
|
||||||
|
}
|
||||||
|
|
||||||
func TestLsd(t *testing.T) {
|
func TestLsd(t *testing.T) {
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
err := fs.ListDir(fremote, &buf)
|
err := fs.ListDir(fremote, &buf)
|
||||||
|
@ -1,4 +1,8 @@
|
|||||||
// Generic tests for testing the Fs and Object interfaces
|
// Generic tests for testing the Fs and Object interfaces
|
||||||
|
//
|
||||||
|
// Run go generate to write the tests for the remotes
|
||||||
|
|
||||||
|
//go:generate go run gen_tests.go
|
||||||
package fstests
|
package fstests
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@ -247,6 +251,41 @@ func TestFsListFile1and2(t *testing.T) {
|
|||||||
fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
|
fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFsCopy(t *testing.T) {
|
||||||
|
skipIfNotOk(t)
|
||||||
|
|
||||||
|
// Check have Copy
|
||||||
|
_, ok := remote.(fs.Copier)
|
||||||
|
if !ok {
|
||||||
|
t.Skip("FS has no Copier interface")
|
||||||
|
}
|
||||||
|
|
||||||
|
var file1Copy = file1
|
||||||
|
file1Copy.Path += "-copy"
|
||||||
|
|
||||||
|
// do the copy
|
||||||
|
src := findObject(t, file1.Path)
|
||||||
|
dst, err := remote.(fs.Copier).Copy(src, file1Copy.Path)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Copy failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// check file exists in new listing
|
||||||
|
fstest.CheckListing(t, remote, []fstest.Item{file1, file2, file1Copy})
|
||||||
|
|
||||||
|
// Check dst lightly - list above has checked ModTime/Md5sum
|
||||||
|
if dst.Remote() != file1Copy.Path {
|
||||||
|
t.Errorf("object path: want %q got %q", file1Copy.Path, dst.Remote())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete copy
|
||||||
|
err = dst.Remove()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Remove copy error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
func TestFsRmdirFull(t *testing.T) {
|
func TestFsRmdirFull(t *testing.T) {
|
||||||
skipIfNotOk(t)
|
skipIfNotOk(t)
|
||||||
err := remote.Rmdir()
|
err := remote.Rmdir()
|
||||||
|
@ -92,7 +92,7 @@ func generateTestProgram(t *template.Template, fns []string, Fsname string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
data := Data{
|
data := Data{
|
||||||
Regenerate: "go run gen_tests.go or make gen_tests",
|
Regenerate: "make gen_tests",
|
||||||
FsName: fsname,
|
FsName: fsname,
|
||||||
UpperFsName: Fsname,
|
UpperFsName: Fsname,
|
||||||
TestName: TestName,
|
TestName: TestName,
|
||||||
|
@ -401,6 +401,38 @@ func (fs *FsStorage) Precision() time.Duration {
|
|||||||
return time.Nanosecond
|
return time.Nanosecond
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Copy src to this remote using server side copy operations.
|
||||||
|
//
|
||||||
|
// This is stored with the remote path given
|
||||||
|
//
|
||||||
|
// It returns the destination Object and a possible error
|
||||||
|
//
|
||||||
|
// Will only be called if src.Fs().Name() == f.Name()
|
||||||
|
//
|
||||||
|
// If it isn't possible then return fs.ErrorCantCopy
|
||||||
|
func (f *FsStorage) Copy(src fs.Object, remote string) (fs.Object, error) {
|
||||||
|
srcObj, ok := src.(*FsObjectStorage)
|
||||||
|
if !ok {
|
||||||
|
fs.Debug(src, "Can't copy - not same remote type")
|
||||||
|
return nil, fs.ErrorCantCopy
|
||||||
|
}
|
||||||
|
|
||||||
|
// Temporary FsObject under construction
|
||||||
|
dstObj := &FsObjectStorage{storage: f, remote: remote}
|
||||||
|
|
||||||
|
srcBucket := srcObj.storage.bucket
|
||||||
|
srcObject := srcObj.storage.root + srcObj.remote
|
||||||
|
dstBucket := f.bucket
|
||||||
|
dstObject := f.root + remote
|
||||||
|
newObject, err := f.svc.Objects.Copy(srcBucket, srcObject, dstBucket, dstObject, nil).Do()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Set the metadata for the new object while we have it
|
||||||
|
dstObj.setMetaData(newObject)
|
||||||
|
return dstObj, nil
|
||||||
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Return the parent Fs
|
// Return the parent Fs
|
||||||
@ -578,4 +610,5 @@ func (o *FsObjectStorage) Remove() error {
|
|||||||
|
|
||||||
// Check the interfaces are satisfied
|
// Check the interfaces are satisfied
|
||||||
var _ fs.Fs = &FsStorage{}
|
var _ fs.Fs = &FsStorage{}
|
||||||
|
var _ fs.Copier = &FsStorage{}
|
||||||
var _ fs.Object = &FsObjectStorage{}
|
var _ fs.Object = &FsObjectStorage{}
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
// Test GoogleCloudStorage filesystem interface
|
// Test GoogleCloudStorage filesystem interface
|
||||||
//
|
//
|
||||||
// Automatically generated - DO NOT EDIT
|
// Automatically generated - DO NOT EDIT
|
||||||
// Regenerate with: go run gen_tests.go or make gen_tests
|
// Regenerate with: make gen_tests
|
||||||
package googlecloudstorage_test
|
package googlecloudstorage_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
|
|||||||
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
||||||
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
||||||
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
||||||
|
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
|
||||||
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
||||||
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
||||||
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
// Test Local filesystem interface
|
// Test Local filesystem interface
|
||||||
//
|
//
|
||||||
// Automatically generated - DO NOT EDIT
|
// Automatically generated - DO NOT EDIT
|
||||||
// Regenerate with: go run gen_tests.go or make gen_tests
|
// Regenerate with: make gen_tests
|
||||||
package local_test
|
package local_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
|
|||||||
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
||||||
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
||||||
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
||||||
|
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
|
||||||
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
||||||
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
||||||
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
||||||
|
32
s3/s3.go
32
s3/s3.go
@ -482,6 +482,37 @@ func (f *FsS3) Precision() time.Duration {
|
|||||||
return time.Nanosecond
|
return time.Nanosecond
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Copy src to this remote using server side copy operations.
|
||||||
|
//
|
||||||
|
// This is stored with the remote path given
|
||||||
|
//
|
||||||
|
// It returns the destination Object and a possible error
|
||||||
|
//
|
||||||
|
// Will only be called if src.Fs().Name() == f.Name()
|
||||||
|
//
|
||||||
|
// If it isn't possible then return fs.ErrorCantCopy
|
||||||
|
func (f *FsS3) Copy(src fs.Object, remote string) (fs.Object, error) {
|
||||||
|
srcObj, ok := src.(*FsObjectS3)
|
||||||
|
if !ok {
|
||||||
|
fs.Debug(src, "Can't copy - not same remote type")
|
||||||
|
return nil, fs.ErrorCantCopy
|
||||||
|
}
|
||||||
|
srcFs := srcObj.s3
|
||||||
|
key := f.root + remote
|
||||||
|
source := srcFs.bucket + "/" + srcFs.root + srcObj.remote
|
||||||
|
req := s3.CopyObjectInput{
|
||||||
|
Bucket: &f.bucket,
|
||||||
|
Key: &key,
|
||||||
|
CopySource: &source,
|
||||||
|
MetadataDirective: aws.String(s3.MetadataDirectiveCopy),
|
||||||
|
}
|
||||||
|
_, err := f.c.CopyObject(&req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return f.NewFsObject(remote), err
|
||||||
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Return the parent Fs
|
// Return the parent Fs
|
||||||
@ -679,4 +710,5 @@ func (o *FsObjectS3) Remove() error {
|
|||||||
|
|
||||||
// Check the interfaces are satisfied
|
// Check the interfaces are satisfied
|
||||||
var _ fs.Fs = &FsS3{}
|
var _ fs.Fs = &FsS3{}
|
||||||
|
var _ fs.Copier = &FsS3{}
|
||||||
var _ fs.Object = &FsObjectS3{}
|
var _ fs.Object = &FsObjectS3{}
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
// Test S3 filesystem interface
|
// Test S3 filesystem interface
|
||||||
//
|
//
|
||||||
// Automatically generated - DO NOT EDIT
|
// Automatically generated - DO NOT EDIT
|
||||||
// Regenerate with: go run gen_tests.go or make gen_tests
|
// Regenerate with: make gen_tests
|
||||||
package s3_test
|
package s3_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
|
|||||||
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
||||||
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
||||||
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
||||||
|
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
|
||||||
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
||||||
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
||||||
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
||||||
|
@ -328,6 +328,29 @@ func (fs *FsSwift) Precision() time.Duration {
|
|||||||
return time.Nanosecond
|
return time.Nanosecond
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Copy src to this remote using server side copy operations.
|
||||||
|
//
|
||||||
|
// This is stored with the remote path given
|
||||||
|
//
|
||||||
|
// It returns the destination Object and a possible error
|
||||||
|
//
|
||||||
|
// Will only be called if src.Fs().Name() == f.Name()
|
||||||
|
//
|
||||||
|
// If it isn't possible then return fs.ErrorCantCopy
|
||||||
|
func (f *FsSwift) Copy(src fs.Object, remote string) (fs.Object, error) {
|
||||||
|
srcObj, ok := src.(*FsObjectSwift)
|
||||||
|
if !ok {
|
||||||
|
fs.Debug(src, "Can't copy - not same remote type")
|
||||||
|
return nil, fs.ErrorCantCopy
|
||||||
|
}
|
||||||
|
srcFs := srcObj.swift
|
||||||
|
_, err := f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return f.NewFsObject(remote), nil
|
||||||
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
// ------------------------------------------------------------
|
||||||
|
|
||||||
// Return the parent Fs
|
// Return the parent Fs
|
||||||
@ -446,4 +469,5 @@ func (o *FsObjectSwift) Remove() error {
|
|||||||
|
|
||||||
// Check the interfaces are satisfied
|
// Check the interfaces are satisfied
|
||||||
var _ fs.Fs = &FsSwift{}
|
var _ fs.Fs = &FsSwift{}
|
||||||
|
var _ fs.Copier = &FsSwift{}
|
||||||
var _ fs.Object = &FsObjectSwift{}
|
var _ fs.Object = &FsObjectSwift{}
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
// Test Swift filesystem interface
|
// Test Swift filesystem interface
|
||||||
//
|
//
|
||||||
// Automatically generated - DO NOT EDIT
|
// Automatically generated - DO NOT EDIT
|
||||||
// Regenerate with: go run gen_tests.go or make gen_tests
|
// Regenerate with: make gen_tests
|
||||||
package swift_test
|
package swift_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@ -34,6 +34,7 @@ func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) }
|
|||||||
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
|
||||||
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) }
|
||||||
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
|
||||||
|
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
|
||||||
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
|
||||||
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
|
||||||
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
|
||||||
|
Loading…
Reference in New Issue
Block a user