diff --git a/backend/local/local.go b/backend/local/local.go index bcd6d9b68..86de84fa4 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -118,6 +118,16 @@ Windows/macOS and case sensitive for everything else. Use this flag to override the default choice.`, Default: false, Advanced: true, + }, { + Name: "no_sparse", + Help: `Disable sparse files for multi-thread downloads + +On Windows platforms rclone will make sparse files when doing +multi-thread downloads. This avoids long pauses on large files where +the OS zeros the file. However sparse files may be undesirable as they +cause disk fragmentation and can be slow to work with.`, + Default: false, + Advanced: true, }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, @@ -139,6 +149,7 @@ type Options struct { OneFileSystem bool `config:"one_file_system"` CaseSensitive bool `config:"case_sensitive"` CaseInsensitive bool `config:"case_insensitive"` + NoSparse bool `config:"no_sparse"` Enc encoder.MultiEncoder `config:"encoding"` } @@ -1095,6 +1106,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return o.lstat() } +var sparseWarning sync.Once + // OpenWriterAt opens with a handle for random access writes // // Pass in the remote desired and the size if known. @@ -1122,10 +1135,15 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr if err != nil { fs.Debugf(o, "Failed to pre-allocate: %v", err) } - // Set the file to be a sparse file (important on Windows) - err = file.SetSparse(out) - if err != nil { - fs.Debugf(o, "Failed to set sparse: %v", err) + if !f.opt.NoSparse && file.SetSparseImplemented { + sparseWarning.Do(func() { + fs.Infof(nil, "Writing sparse files: use --local-no-sparse or --multi-thread-streams 0 to disable") + }) + // Set the file to be a sparse file (important on Windows) + err = file.SetSparse(out) + if err != nil { + fs.Debugf(o, "Failed to set sparse: %v", err) + } } return out, nil diff --git a/docs/content/docs.md b/docs/content/docs.md index 2d64733e3..3c8f1f0a6 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -844,6 +844,12 @@ with any source. as they are faster without unless `--multi-thread-streams` is set explicitly. +**NB** on Windows using multi-thread downloads will cause the +resulting files to be [sparse](https://en.wikipedia.org/wiki/Sparse_file). +Use `--local-no-sparse` to disable sparse files (which may cause long +delays at the start of downloads) or disable multi-thread downloads +with `--multi-thread-streams 0` + ### --multi-thread-streams=N ### When using multi thread downloads (see above `--multi-thread-cutoff`)