operations: implement --partial-suffix to control extension of temporary file names

This commit is contained in:
Volodymyr 2023-10-10 14:27:32 +03:00 committed by GitHub
parent f56ea2bee2
commit 6255d9dfaa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 26 additions and 3 deletions

View File

@ -1278,8 +1278,9 @@ flag set) such as:
- sftp
Without `--inplace` (the default) rclone will first upload to a
temporary file with an extension like this where `XXXXXX` represents a
random string.
temporary file with an extension like this, where `XXXXXX` represents a
random string and `.partial` is [--partial-suffix](#partial-suffix) value
(`.partial` by default).
original-file-name.XXXXXX.partial
@ -1726,6 +1727,15 @@ If you want perfect ordering then you will need to specify
[--check-first](#check-first) which will find all the files which need
transferring first before transferring any.
### --partial-suffix {#partial-suffix}
When [--inplace](#inplace) is not used, it causes rclone to use
the `--partial-suffix` as suffix for temporary files.
Suffix length limit is 16 characters.
The default is `.partial`.
### --password-command SpaceSepList ###
This flag supplies a program which should supply the config password

View File

@ -148,6 +148,7 @@ type ConfigInfo struct {
TerminalColorMode TerminalColorMode
DefaultTime Time // time that directories with no time should display
Inplace bool // Download directly to destination file instead of atomic download to temp/rename
PartialSuffix string
}
// NewConfig creates a new config with everything set to the default
@ -192,6 +193,7 @@ func NewConfig() *ConfigInfo {
c.FsCacheExpireInterval = 60 * time.Second
c.KvLockTime = 1 * time.Second
c.DefaultTime = Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
c.PartialSuffix = ".partial"
// Perform a simple check for debug flags to enable debug logging during the flag initialization
for argIndex, arg := range os.Args {

View File

@ -38,6 +38,7 @@ var (
downloadHeaders []string
headers []string
metadataSet []string
partialSuffix string
)
// AddFlags adds the non filing system specific flags to the command
@ -148,6 +149,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
flags.FVarP(flagSet, &ci.TerminalColorMode, "color", "", "When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS", "Config")
flags.FVarP(flagSet, &ci.DefaultTime, "default-time", "", "Time to show if modtime is unknown for files and directories", "Config,Listing")
flags.BoolVarP(flagSet, &ci.Inplace, "inplace", "", ci.Inplace, "Download directly to destination file instead of atomic download to temp/rename", "Copy")
flags.StringVarP(flagSet, &partialSuffix, "partial-suffix", "", ci.PartialSuffix, "Add partial-suffix to temporary file name when --inplace is not used", "Copy")
}
// ParseHeaders converts the strings passed in via the header flags into HTTPOptions
@ -322,6 +324,11 @@ func SetFlags(ci *fs.ConfigInfo) {
multiThreadStreamsFlag := pflag.Lookup("multi-thread-streams")
ci.MultiThreadSet = multiThreadStreamsFlag != nil && multiThreadStreamsFlag.Changed
if len(partialSuffix) > 16 {
log.Fatalf("--partial-suffix: Expecting suffix length not greater than %d but got %d", 16, len(partialSuffix))
}
ci.PartialSuffix = partialSuffix
// Make sure some values are > 0
nonZero := func(pi *int) {
if *pi <= 0 {

View File

@ -359,9 +359,13 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
remotePartial = remote
)
if !ci.Inplace && f.Features().Move != nil && f.Features().PartialUploads && !strings.HasSuffix(remote, ".rclonelink") {
if len(ci.PartialSuffix) > 16 {
return nil, fmt.Errorf("expecting length of --partial-suffix to be not greater than %d but got %d", 16, len(ci.PartialSuffix))
}
// Avoid making the leaf name longer if it's already lengthy to avoid
// trouble with file name length limits.
suffix := "." + random.String(8) + ".partial"
suffix := "." + random.String(8) + ci.PartialSuffix
base := path.Base(remotePartial)
if len(base) > 100 {
remotePartial = remotePartial[:len(remotePartial)-len(suffix)] + suffix