extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-01-10 16:28:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

operations: add --server-side-across-configs global flag for any backend

Browse Source
This commit is contained in:
Nick Craig-Wood 2022-07-04 16:26:08 +01:00
parent 9c6cfc1ff0
commit 55bbff6346
4 changed files with 107 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/content/docs.md
View File

@ -1643,6 +1643,18 @@ This sets the interval between each retry specified by `--retries`
The default is `0`. Use `0` to disable. The default is `0`. Use `0` to disable.
### --server-side-across-configs ###
Allow server-side operations (e.g. copy or move) to work across
different configurations.
This can be useful if you wish to do a server-side copy or move
between two remotes which use the same backend but are configured
differently.
Note that this isn't enabled by default because it isn't easy for
rclone to tell if it will work between any two configurations.
### --size-only ### ### --size-only ###
Normally rclone will look at modification time and size of files to Normally rclone will look at modification time and size of files to

181
fs/config.go
View File

@ -45,96 +45,97 @@ var (
// ConfigInfo is filesystem config options // ConfigInfo is filesystem config options
type ConfigInfo struct { type ConfigInfo struct {
LogLevel LogLevel LogLevel LogLevel
StatsLogLevel LogLevel StatsLogLevel LogLevel
UseJSONLog bool UseJSONLog bool
DryRun bool DryRun bool
Interactive bool Interactive bool
CheckSum bool CheckSum bool
SizeOnly bool SizeOnly bool
IgnoreTimes bool IgnoreTimes bool
IgnoreExisting bool IgnoreExisting bool
IgnoreErrors bool IgnoreErrors bool
ModifyWindow time.Duration ModifyWindow time.Duration
Checkers int Checkers int
Transfers int Transfers int
ConnectTimeout time.Duration // Connect timeout ConnectTimeout time.Duration // Connect timeout
Timeout time.Duration // Data channel timeout Timeout time.Duration // Data channel timeout
ExpectContinueTimeout time.Duration ExpectContinueTimeout time.Duration
Dump DumpFlags Dump DumpFlags
InsecureSkipVerify bool // Skip server certificate verification InsecureSkipVerify bool // Skip server certificate verification
DeleteMode DeleteMode DeleteMode DeleteMode
MaxDelete int64 MaxDelete int64
TrackRenames bool // Track file renames. TrackRenames bool // Track file renames.
TrackRenamesStrategy string // Comma separated list of strategies used to track renames TrackRenamesStrategy string // Comma separated list of strategies used to track renames
LowLevelRetries int LowLevelRetries int
UpdateOlder bool // Skip files that are newer on the destination UpdateOlder bool // Skip files that are newer on the destination
NoGzip bool // Disable compression NoGzip bool // Disable compression
MaxDepth int MaxDepth int
IgnoreSize bool IgnoreSize bool
IgnoreChecksum bool IgnoreChecksum bool
IgnoreCaseSync bool IgnoreCaseSync bool
NoTraverse bool NoTraverse bool
CheckFirst bool CheckFirst bool
NoCheckDest bool NoCheckDest bool
NoUnicodeNormalization bool NoUnicodeNormalization bool
NoUpdateModTime bool NoUpdateModTime bool
DataRateUnit string DataRateUnit string
CompareDest []string CompareDest []string
CopyDest []string CopyDest []string
BackupDir string BackupDir string
Suffix string Suffix string
SuffixKeepExtension bool SuffixKeepExtension bool
UseListR bool UseListR bool
BufferSize SizeSuffix BufferSize SizeSuffix
BwLimit BwTimetable BwLimit BwTimetable
BwLimitFile BwTimetable BwLimitFile BwTimetable
TPSLimit float64 TPSLimit float64
TPSLimitBurst int TPSLimitBurst int
BindAddr net.IP BindAddr net.IP
DisableFeatures []string DisableFeatures []string
UserAgent string UserAgent string
Immutable bool Immutable bool
AutoConfirm bool AutoConfirm bool
StreamingUploadCutoff SizeSuffix StreamingUploadCutoff SizeSuffix
StatsFileNameLength int StatsFileNameLength int
AskPassword bool AskPassword bool
PasswordCommand SpaceSepList PasswordCommand SpaceSepList
UseServerModTime bool UseServerModTime bool
MaxTransfer SizeSuffix MaxTransfer SizeSuffix
MaxDuration time.Duration MaxDuration time.Duration
CutoffMode CutoffMode CutoffMode CutoffMode
MaxBacklog int MaxBacklog int
MaxStatsGroups int MaxStatsGroups int
StatsOneLine bool StatsOneLine bool
StatsOneLineDate bool // If we want a date prefix at all StatsOneLineDate bool // If we want a date prefix at all
StatsOneLineDateFormat string // If we want to customize the prefix StatsOneLineDateFormat string // If we want to customize the prefix
ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred
Progress bool Progress bool
ProgressTerminalTitle bool ProgressTerminalTitle bool
Cookie bool Cookie bool
UseMmap bool UseMmap bool
CaCert string // Client Side CA CaCert string // Client Side CA
ClientCert string // Client Side Cert ClientCert string // Client Side Cert
ClientKey string // Client Side Key ClientKey string // Client Side Key
MultiThreadCutoff SizeSuffix MultiThreadCutoff SizeSuffix
MultiThreadStreams int MultiThreadStreams int
MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags) MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags)
OrderBy string // instructions on how to order the transfer OrderBy string // instructions on how to order the transfer
UploadHeaders []*HTTPOption UploadHeaders []*HTTPOption
DownloadHeaders []*HTTPOption DownloadHeaders []*HTTPOption
Headers []*HTTPOption Headers []*HTTPOption
MetadataSet Metadata // extra metadata to write when uploading MetadataSet Metadata // extra metadata to write when uploading
RefreshTimes bool RefreshTimes bool
NoConsole bool NoConsole bool
TrafficClass uint8 TrafficClass uint8
FsCacheExpireDuration time.Duration FsCacheExpireDuration time.Duration
FsCacheExpireInterval time.Duration FsCacheExpireInterval time.Duration
DisableHTTP2 bool DisableHTTP2 bool
HumanReadable bool HumanReadable bool
KvLockTime time.Duration // maximum time to keep key-value database locked by process KvLockTime time.Duration // maximum time to keep key-value database locked by process
DisableHTTPKeepAlives bool DisableHTTPKeepAlives bool
Metadata bool Metadata bool
ServerSideAcrossConfigs bool
} }
// NewConfig creates a new config with everything set to the default // NewConfig creates a new config with everything set to the default

1
fs/config/configflags/configflags.go
View File

@ -141,6 +141,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
flags.DurationVarP(flagSet, &ci.KvLockTime, "kv-lock-time", "", ci.KvLockTime, "Maximum time to keep key-value database locked by process") flags.DurationVarP(flagSet, &ci.KvLockTime, "kv-lock-time", "", ci.KvLockTime, "Maximum time to keep key-value database locked by process")
flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.") flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.")
flags.BoolVarP(flagSet, &ci.Metadata, "metadata", "M", ci.Metadata, "If set, preserve metadata when copying objects") flags.BoolVarP(flagSet, &ci.Metadata, "metadata", "M", ci.Metadata, "If set, preserve metadata when copying objects")
flags.BoolVarP(flagSet, &ci.ServerSideAcrossConfigs, "server-side-across-configs", "", ci.ServerSideAcrossConfigs, "Allow server-side operations (e.g. copy) to work across different configs")
} }
// ParseHeaders converts the strings passed in via the header flags into HTTPOptions // ParseHeaders converts the strings passed in via the header flags into HTTPOptions

5
fs/operations/operations.go
View File

@ -424,7 +424,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
return nil, accounting.ErrorMaxTransferLimitReachedGraceful return nil, accounting.ErrorMaxTransferLimitReachedGraceful
} }
} }
if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && f.Features().ServerSideAcrossConfigs)) { if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && (f.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) {
in := tr.Account(ctx, nil) // account the transfer in := tr.Account(ctx, nil) // account the transfer
in.ServerSideCopyStart() in.ServerSideCopyStart()
newDst, err = doCopy(ctx, src, remote) newDst, err = doCopy(ctx, src, remote)
@ -604,6 +604,7 @@ func SameObject(src, dst fs.Object) bool {
// It returns the destination object if possible. Note that this may // It returns the destination object if possible. Note that this may
// be nil. // be nil.
func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) { func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
ci := fs.GetConfig(ctx)
tr := accounting.Stats(ctx).NewCheckingTransfer(src) tr := accounting.Stats(ctx).NewCheckingTransfer(src)
defer func() { defer func() {
if err == nil { if err == nil {
@ -618,7 +619,7 @@ func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
return newDst, nil return newDst, nil
} }
// See if we have Move available // See if we have Move available
if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && fdst.Features().ServerSideAcrossConfigs)) { if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) {
// Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive) // Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive)
if dst != nil && !SameObject(src, dst) { if dst != nil && !SameObject(src, dst) {
err = DeleteFile(ctx, dst) err = DeleteFile(ctx, dst)