From 55bbff6346bb2a1ee3798844860a92fee7636918 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 4 Jul 2022 16:26:08 +0100 Subject: [PATCH] operations: add --server-side-across-configs global flag for any backend --- docs/content/docs.md | 12 ++ fs/config.go | 181 ++++++++++++++------------- fs/config/configflags/configflags.go | 1 + fs/operations/operations.go | 5 +- 4 files changed, 107 insertions(+), 92 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index a71728cc7..cefa65c23 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -1643,6 +1643,18 @@ This sets the interval between each retry specified by `--retries` The default is `0`. Use `0` to disable. +### --server-side-across-configs ### + +Allow server-side operations (e.g. copy or move) to work across +different configurations. + +This can be useful if you wish to do a server-side copy or move +between two remotes which use the same backend but are configured +differently. + +Note that this isn't enabled by default because it isn't easy for +rclone to tell if it will work between any two configurations. + ### --size-only ### Normally rclone will look at modification time and size of files to diff --git a/fs/config.go b/fs/config.go index a39779bae..b253a5c30 100644 --- a/fs/config.go +++ b/fs/config.go @@ -45,96 +45,97 @@ var ( // ConfigInfo is filesystem config options type ConfigInfo struct { - LogLevel LogLevel - StatsLogLevel LogLevel - UseJSONLog bool - DryRun bool - Interactive bool - CheckSum bool - SizeOnly bool - IgnoreTimes bool - IgnoreExisting bool - IgnoreErrors bool - ModifyWindow time.Duration - Checkers int - Transfers int - ConnectTimeout time.Duration // Connect timeout - Timeout time.Duration // Data channel timeout - ExpectContinueTimeout time.Duration - Dump DumpFlags - InsecureSkipVerify bool // Skip server certificate verification - DeleteMode DeleteMode - MaxDelete int64 - TrackRenames bool // Track file renames. - TrackRenamesStrategy string // Comma separated list of strategies used to track renames - LowLevelRetries int - UpdateOlder bool // Skip files that are newer on the destination - NoGzip bool // Disable compression - MaxDepth int - IgnoreSize bool - IgnoreChecksum bool - IgnoreCaseSync bool - NoTraverse bool - CheckFirst bool - NoCheckDest bool - NoUnicodeNormalization bool - NoUpdateModTime bool - DataRateUnit string - CompareDest []string - CopyDest []string - BackupDir string - Suffix string - SuffixKeepExtension bool - UseListR bool - BufferSize SizeSuffix - BwLimit BwTimetable - BwLimitFile BwTimetable - TPSLimit float64 - TPSLimitBurst int - BindAddr net.IP - DisableFeatures []string - UserAgent string - Immutable bool - AutoConfirm bool - StreamingUploadCutoff SizeSuffix - StatsFileNameLength int - AskPassword bool - PasswordCommand SpaceSepList - UseServerModTime bool - MaxTransfer SizeSuffix - MaxDuration time.Duration - CutoffMode CutoffMode - MaxBacklog int - MaxStatsGroups int - StatsOneLine bool - StatsOneLineDate bool // If we want a date prefix at all - StatsOneLineDateFormat string // If we want to customize the prefix - ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred - Progress bool - ProgressTerminalTitle bool - Cookie bool - UseMmap bool - CaCert string // Client Side CA - ClientCert string // Client Side Cert - ClientKey string // Client Side Key - MultiThreadCutoff SizeSuffix - MultiThreadStreams int - MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags) - OrderBy string // instructions on how to order the transfer - UploadHeaders []*HTTPOption - DownloadHeaders []*HTTPOption - Headers []*HTTPOption - MetadataSet Metadata // extra metadata to write when uploading - RefreshTimes bool - NoConsole bool - TrafficClass uint8 - FsCacheExpireDuration time.Duration - FsCacheExpireInterval time.Duration - DisableHTTP2 bool - HumanReadable bool - KvLockTime time.Duration // maximum time to keep key-value database locked by process - DisableHTTPKeepAlives bool - Metadata bool + LogLevel LogLevel + StatsLogLevel LogLevel + UseJSONLog bool + DryRun bool + Interactive bool + CheckSum bool + SizeOnly bool + IgnoreTimes bool + IgnoreExisting bool + IgnoreErrors bool + ModifyWindow time.Duration + Checkers int + Transfers int + ConnectTimeout time.Duration // Connect timeout + Timeout time.Duration // Data channel timeout + ExpectContinueTimeout time.Duration + Dump DumpFlags + InsecureSkipVerify bool // Skip server certificate verification + DeleteMode DeleteMode + MaxDelete int64 + TrackRenames bool // Track file renames. + TrackRenamesStrategy string // Comma separated list of strategies used to track renames + LowLevelRetries int + UpdateOlder bool // Skip files that are newer on the destination + NoGzip bool // Disable compression + MaxDepth int + IgnoreSize bool + IgnoreChecksum bool + IgnoreCaseSync bool + NoTraverse bool + CheckFirst bool + NoCheckDest bool + NoUnicodeNormalization bool + NoUpdateModTime bool + DataRateUnit string + CompareDest []string + CopyDest []string + BackupDir string + Suffix string + SuffixKeepExtension bool + UseListR bool + BufferSize SizeSuffix + BwLimit BwTimetable + BwLimitFile BwTimetable + TPSLimit float64 + TPSLimitBurst int + BindAddr net.IP + DisableFeatures []string + UserAgent string + Immutable bool + AutoConfirm bool + StreamingUploadCutoff SizeSuffix + StatsFileNameLength int + AskPassword bool + PasswordCommand SpaceSepList + UseServerModTime bool + MaxTransfer SizeSuffix + MaxDuration time.Duration + CutoffMode CutoffMode + MaxBacklog int + MaxStatsGroups int + StatsOneLine bool + StatsOneLineDate bool // If we want a date prefix at all + StatsOneLineDateFormat string // If we want to customize the prefix + ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred + Progress bool + ProgressTerminalTitle bool + Cookie bool + UseMmap bool + CaCert string // Client Side CA + ClientCert string // Client Side Cert + ClientKey string // Client Side Key + MultiThreadCutoff SizeSuffix + MultiThreadStreams int + MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags) + OrderBy string // instructions on how to order the transfer + UploadHeaders []*HTTPOption + DownloadHeaders []*HTTPOption + Headers []*HTTPOption + MetadataSet Metadata // extra metadata to write when uploading + RefreshTimes bool + NoConsole bool + TrafficClass uint8 + FsCacheExpireDuration time.Duration + FsCacheExpireInterval time.Duration + DisableHTTP2 bool + HumanReadable bool + KvLockTime time.Duration // maximum time to keep key-value database locked by process + DisableHTTPKeepAlives bool + Metadata bool + ServerSideAcrossConfigs bool } // NewConfig creates a new config with everything set to the default diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 61aa65f91..42b2b4d05 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -141,6 +141,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { flags.DurationVarP(flagSet, &ci.KvLockTime, "kv-lock-time", "", ci.KvLockTime, "Maximum time to keep key-value database locked by process") flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.") flags.BoolVarP(flagSet, &ci.Metadata, "metadata", "M", ci.Metadata, "If set, preserve metadata when copying objects") + flags.BoolVarP(flagSet, &ci.ServerSideAcrossConfigs, "server-side-across-configs", "", ci.ServerSideAcrossConfigs, "Allow server-side operations (e.g. copy) to work across different configs") } // ParseHeaders converts the strings passed in via the header flags into HTTPOptions diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 0b0f9f75e..fc275c894 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -424,7 +424,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj return nil, accounting.ErrorMaxTransferLimitReachedGraceful } } - if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && f.Features().ServerSideAcrossConfigs)) { + if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && (f.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) { in := tr.Account(ctx, nil) // account the transfer in.ServerSideCopyStart() newDst, err = doCopy(ctx, src, remote) @@ -604,6 +604,7 @@ func SameObject(src, dst fs.Object) bool { // It returns the destination object if possible. Note that this may // be nil. func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) { + ci := fs.GetConfig(ctx) tr := accounting.Stats(ctx).NewCheckingTransfer(src) defer func() { if err == nil { @@ -618,7 +619,7 @@ func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs. return newDst, nil } // See if we have Move available - if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && fdst.Features().ServerSideAcrossConfigs)) { + if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) { // Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive) if dst != nil && !SameObject(src, dst) { err = DeleteFile(ctx, dst)