extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-12-23 15:38:57 +01:00
Code Issues Packages Projects Releases Wiki Activity

local: implement --local-no-sparse flag for disabling sparse files #2469

Browse Source 
This also introduces a one time warning for sparse files and updates
the docs to warn about them.
This commit is contained in:
Nick Craig-Wood 2020-05-19 10:16:43 +01:00
parent 5cb2a2fa3c
commit 610f40f700
2 changed files with 28 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
backend/local/local.go
View File

@ -118,6 +118,16 @@ Windows/macOS and case sensitive for everything else. Use this flag
to override the default choice.`, to override the default choice.`,
Default: false, Default: false,
Advanced: true, Advanced: true,
}, {
Name: "no_sparse",
Help: `Disable sparse files for multi-thread downloads
On Windows platforms rclone will make sparse files when doing
multi-thread downloads. This avoids long pauses on large files where
the OS zeros the file. However sparse files may be undesirable as they
cause disk fragmentation and can be slow to work with.`,
Default: false,
Advanced: true,
}, { }, {
Name: config.ConfigEncoding, Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp, Help: config.ConfigEncodingHelp,
@ -139,6 +149,7 @@ type Options struct {
OneFileSystem bool `config:"one_file_system"` OneFileSystem bool `config:"one_file_system"`
CaseSensitive bool `config:"case_sensitive"` CaseSensitive bool `config:"case_sensitive"`
CaseInsensitive bool `config:"case_insensitive"` CaseInsensitive bool `config:"case_insensitive"`
NoSparse bool `config:"no_sparse"`
Enc encoder.MultiEncoder `config:"encoding"` Enc encoder.MultiEncoder `config:"encoding"`
} }
@ -1095,6 +1106,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return o.lstat() return o.lstat()
} }
var sparseWarning sync.Once
// OpenWriterAt opens with a handle for random access writes // OpenWriterAt opens with a handle for random access writes
// //
// Pass in the remote desired and the size if known. // Pass in the remote desired and the size if known.
@ -1122,10 +1135,15 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
if err != nil { if err != nil {
fs.Debugf(o, "Failed to pre-allocate: %v", err) fs.Debugf(o, "Failed to pre-allocate: %v", err)
} }
// Set the file to be a sparse file (important on Windows) if !f.opt.NoSparse && file.SetSparseImplemented {
err = file.SetSparse(out) sparseWarning.Do(func() {
if err != nil { fs.Infof(nil, "Writing sparse files: use --local-no-sparse or --multi-thread-streams 0 to disable")
fs.Debugf(o, "Failed to set sparse: %v", err) })
// Set the file to be a sparse file (important on Windows)
err = file.SetSparse(out)
if err != nil {
fs.Debugf(o, "Failed to set sparse: %v", err)
}
} }
return out, nil return out, nil

6
docs/content/docs.md
View File

@ -844,6 +844,12 @@ with any source.
as they are faster without unless `--multi-thread-streams` is set as they are faster without unless `--multi-thread-streams` is set
explicitly. explicitly.
**NB** on Windows using multi-thread downloads will cause the
resulting files to be [sparse](https://en.wikipedia.org/wiki/Sparse_file).
Use `--local-no-sparse` to disable sparse files (which may cause long
delays at the start of downloads) or disable multi-thread downloads
with `--multi-thread-streams 0`
### --multi-thread-streams=N ### ### --multi-thread-streams=N ###
When using multi thread downloads (see above `--multi-thread-cutoff`) When using multi thread downloads (see above `--multi-thread-cutoff`)