local: --local-no-clone flag to disable cloning for server-side copies

This flag allows users to disable the reflink cloning feature and instead force
"deep" copies, for certain use cases where data redundancy is preferable. It is
functionally equivalent to using `--disable Copy` on local.
This commit is contained in:
nielash 2024-08-13 02:58:54 -04:00 committed by Nick Craig-Wood
parent f6d836eefd
commit bd5199910b
2 changed files with 148 additions and 104 deletions

View File

@ -22,7 +22,7 @@ import (
// //
// If it isn't possible then return fs.ErrorCantCopy // If it isn't possible then return fs.ErrorCantCopy
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
if runtime.GOOS != "darwin" || f.opt.TranslateSymlinks { if runtime.GOOS != "darwin" || f.opt.TranslateSymlinks || f.opt.NoClone {
return nil, fs.ErrorCantCopy return nil, fs.ErrorCantCopy
} }
srcObj, ok := src.(*Object) srcObj, ok := src.(*Object)

View File

@ -32,9 +32,11 @@ import (
) )
// Constants // Constants
const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset const (
const linkSuffix = ".rclonelink" // The suffix added to a translated symbolic link devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
const useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly linkSuffix = ".rclonelink" // The suffix added to a translated symbolic link
useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
)
// timeType allows the user to choose what exactly ModTime() returns // timeType allows the user to choose what exactly ModTime() returns
type timeType = fs.Enum[timeTypeChoices] type timeType = fs.Enum[timeTypeChoices]
@ -78,41 +80,46 @@ supported by all file systems) under the "user.*" prefix.
Metadata is supported on files and directories. Metadata is supported on files and directories.
`, `,
}, },
Options: []fs.Option{{ Options: []fs.Option{
Name: "nounc", {
Help: "Disable UNC (long path names) conversion on Windows.", Name: "nounc",
Default: false, Help: "Disable UNC (long path names) conversion on Windows.",
Advanced: runtime.GOOS != "windows", Default: false,
Examples: []fs.OptionExample{{ Advanced: runtime.GOOS != "windows",
Value: "true", Examples: []fs.OptionExample{{
Help: "Disables long file names.", Value: "true",
}}, Help: "Disables long file names.",
}, { }},
Name: "copy_links", },
Help: "Follow symlinks and copy the pointed to item.", {
Default: false, Name: "copy_links",
NoPrefix: true, Help: "Follow symlinks and copy the pointed to item.",
ShortOpt: "L", Default: false,
Advanced: true, NoPrefix: true,
}, { ShortOpt: "L",
Name: "links", Advanced: true,
Help: "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension.", },
Default: false, {
NoPrefix: true, Name: "links",
ShortOpt: "l", Help: "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension.",
Advanced: true, Default: false,
}, { NoPrefix: true,
Name: "skip_links", ShortOpt: "l",
Help: `Don't warn about skipped symlinks. Advanced: true,
},
{
Name: "skip_links",
Help: `Don't warn about skipped symlinks.
This flag disables warning messages on skipped symlinks or junction This flag disables warning messages on skipped symlinks or junction
points, as you explicitly acknowledge that they should be skipped.`, points, as you explicitly acknowledge that they should be skipped.`,
Default: false, Default: false,
NoPrefix: true, NoPrefix: true,
Advanced: true, Advanced: true,
}, { },
Name: "zero_size_links", {
Help: `Assume the Stat size of links is zero (and read them instead) (deprecated). Name: "zero_size_links",
Help: `Assume the Stat size of links is zero (and read them instead) (deprecated).
Rclone used to use the Stat size of links as the link size, but this fails in quite a few places: Rclone used to use the Stat size of links as the link size, but this fails in quite a few places:
@ -122,11 +129,12 @@ Rclone used to use the Stat size of links as the link size, but this fails in qu
So rclone now always reads the link. So rclone now always reads the link.
`, `,
Default: false, Default: false,
Advanced: true, Advanced: true,
}, { },
Name: "unicode_normalization", {
Help: `Apply unicode NFC normalization to paths and filenames. Name: "unicode_normalization",
Help: `Apply unicode NFC normalization to paths and filenames.
This flag can be used to normalize file names into unicode NFC form This flag can be used to normalize file names into unicode NFC form
that are read from the local filesystem. that are read from the local filesystem.
@ -140,11 +148,12 @@ some OSes.
Note that rclone compares filenames with unicode normalization in the sync Note that rclone compares filenames with unicode normalization in the sync
routine so this flag shouldn't normally be used.`, routine so this flag shouldn't normally be used.`,
Default: false, Default: false,
Advanced: true, Advanced: true,
}, { },
Name: "no_check_updated", {
Help: `Don't check to see if the files change during upload. Name: "no_check_updated",
Help: `Don't check to see if the files change during upload.
Normally rclone checks the size and modification time of files as they Normally rclone checks the size and modification time of files as they
are being uploaded and aborts with a message which starts "can't copy - are being uploaded and aborts with a message which starts "can't copy -
@ -175,68 +184,96 @@ directory listing (where the initial stat value comes from on Windows)
and when stat is called on them directly. Other copy tools always use and when stat is called on them directly. Other copy tools always use
the direct stat value and setting this flag will disable that. the direct stat value and setting this flag will disable that.
`, `,
Default: false, Default: false,
Advanced: true, Advanced: true,
}, { },
Name: "one_file_system", {
Help: "Don't cross filesystem boundaries (unix/macOS only).", Name: "one_file_system",
Default: false, Help: "Don't cross filesystem boundaries (unix/macOS only).",
NoPrefix: true, Default: false,
ShortOpt: "x", NoPrefix: true,
Advanced: true, ShortOpt: "x",
}, { Advanced: true,
Name: "case_sensitive", },
Help: `Force the filesystem to report itself as case sensitive. {
Name: "case_sensitive",
Help: `Force the filesystem to report itself as case sensitive.
Normally the local backend declares itself as case insensitive on Normally the local backend declares itself as case insensitive on
Windows/macOS and case sensitive for everything else. Use this flag Windows/macOS and case sensitive for everything else. Use this flag
to override the default choice.`, to override the default choice.`,
Default: false, Default: false,
Advanced: true, Advanced: true,
}, { },
Name: "case_insensitive", {
Help: `Force the filesystem to report itself as case insensitive. Name: "case_insensitive",
Help: `Force the filesystem to report itself as case insensitive.
Normally the local backend declares itself as case insensitive on Normally the local backend declares itself as case insensitive on
Windows/macOS and case sensitive for everything else. Use this flag Windows/macOS and case sensitive for everything else. Use this flag
to override the default choice.`, to override the default choice.`,
Default: false, Default: false,
Advanced: true, Advanced: true,
}, { },
Name: "no_preallocate", {
Help: `Disable preallocation of disk space for transferred files. Name: "no_clone",
Help: `Disable reflink cloning for server-side copies.
Normally, for local-to-local transfers, rclone will "clone" the file when
possible, and fall back to "copying" only when cloning is not supported.
Cloning creates a shallow copy (or "reflink") which initially shares blocks with
the original file. Unlike a "hardlink", the two files are independent and
neither will affect the other if subsequently modified.
Cloning is usually preferable to copying, as it is much faster and is
deduplicated by default (i.e. having two identical files does not consume more
storage than having just one.) However, for use cases where data redundancy is
preferable, --local-no-clone can be used to disable cloning and force "deep" copies.
Currently, cloning is only supported when using APFS on macOS (support for other
platforms may be added in the future.)`,
Default: false,
Advanced: true,
},
{
Name: "no_preallocate",
Help: `Disable preallocation of disk space for transferred files.
Preallocation of disk space helps prevent filesystem fragmentation. Preallocation of disk space helps prevent filesystem fragmentation.
However, some virtual filesystem layers (such as Google Drive File However, some virtual filesystem layers (such as Google Drive File
Stream) may incorrectly set the actual file size equal to the Stream) may incorrectly set the actual file size equal to the
preallocated space, causing checksum and file size checks to fail. preallocated space, causing checksum and file size checks to fail.
Use this flag to disable preallocation.`, Use this flag to disable preallocation.`,
Default: false, Default: false,
Advanced: true, Advanced: true,
}, { },
Name: "no_sparse", {
Help: `Disable sparse files for multi-thread downloads. Name: "no_sparse",
Help: `Disable sparse files for multi-thread downloads.
On Windows platforms rclone will make sparse files when doing On Windows platforms rclone will make sparse files when doing
multi-thread downloads. This avoids long pauses on large files where multi-thread downloads. This avoids long pauses on large files where
the OS zeros the file. However sparse files may be undesirable as they the OS zeros the file. However sparse files may be undesirable as they
cause disk fragmentation and can be slow to work with.`, cause disk fragmentation and can be slow to work with.`,
Default: false, Default: false,
Advanced: true, Advanced: true,
}, { },
Name: "no_set_modtime", {
Help: `Disable setting modtime. Name: "no_set_modtime",
Help: `Disable setting modtime.
Normally rclone updates modification time of files after they are done Normally rclone updates modification time of files after they are done
uploading. This can cause permissions issues on Linux platforms when uploading. This can cause permissions issues on Linux platforms when
the user rclone is running as does not own the file uploaded, such as the user rclone is running as does not own the file uploaded, such as
when copying to a CIFS mount owned by another user. If this option is when copying to a CIFS mount owned by another user. If this option is
enabled, rclone will no longer update the modtime after copying a file.`, enabled, rclone will no longer update the modtime after copying a file.`,
Default: false, Default: false,
Advanced: true, Advanced: true,
}, { },
Name: "time_type", {
Help: `Set what kind of time is returned. Name: "time_type",
Help: `Set what kind of time is returned.
Normally rclone does all operations on the mtime or Modification time. Normally rclone does all operations on the mtime or Modification time.
@ -255,27 +292,29 @@ will silently replace it with the modification time which all OSes support.
Note that setting the time will still set the modified time so this is Note that setting the time will still set the modified time so this is
only useful for reading. only useful for reading.
`, `,
Default: mTime, Default: mTime,
Advanced: true, Advanced: true,
Examples: []fs.OptionExample{{ Examples: []fs.OptionExample{{
Value: mTime.String(), Value: mTime.String(),
Help: "The last modification time.", Help: "The last modification time.",
}, { }, {
Value: aTime.String(), Value: aTime.String(),
Help: "The last access time.", Help: "The last access time.",
}, { }, {
Value: bTime.String(), Value: bTime.String(),
Help: "The creation time.", Help: "The creation time.",
}, { }, {
Value: cTime.String(), Value: cTime.String(),
Help: "The last status change time.", Help: "The last status change time.",
}}, }},
}, { },
Name: config.ConfigEncoding, {
Help: config.ConfigEncodingHelp, Name: config.ConfigEncoding,
Advanced: true, Help: config.ConfigEncodingHelp,
Default: encoder.OS, Advanced: true,
}}, Default: encoder.OS,
},
},
} }
fs.Register(fsi) fs.Register(fsi)
} }
@ -296,6 +335,7 @@ type Options struct {
NoSetModTime bool `config:"no_set_modtime"` NoSetModTime bool `config:"no_set_modtime"`
TimeType timeType `config:"time_type"` TimeType timeType `config:"time_type"`
Enc encoder.MultiEncoder `config:"encoding"` Enc encoder.MultiEncoder `config:"encoding"`
NoClone bool `config:"no_clone"`
} }
// Fs represents a local filesystem rooted at root // Fs represents a local filesystem rooted at root
@ -384,6 +424,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if opt.FollowSymlinks { if opt.FollowSymlinks {
f.lstat = os.Stat f.lstat = os.Stat
} }
if opt.NoClone {
// Disable server-side copy when --local-no-clone is set
f.features.Copy = nil
}
// Check to see if this points to a file // Check to see if this points to a file
fi, err := f.lstat(f.root) fi, err := f.lstat(f.root)