From 87ec26001f399cd1fe728a98d78c1b6ec4f0d322 Mon Sep 17 00:00:00 2001 From: nielash Date: Thu, 28 Dec 2023 12:30:47 -0500 Subject: [PATCH] local: add server-side copy with xattrs on macOS (part-fix #1710) Before this change, macOS-specific metadata was not preserved by rclone, even for local-to-local transfers (it does not use the "user." prefix, nor is Mac metadata limited to xattrs.) Additionally, rclone did not take advantage of APFS's native "cloning" functionality for fast and deduplicated transfers. After this change, local (on macOS only) supports "server-side copy" similarly to other remotes, and achieves this by using (when possible) macOS's native APFS "cloning", which is the same underlying mechanism deployed when a user duplicates a file via the Finder UI. This has several advantages over the previous behavior: - It is extremely fast (even large files can be cloned instantly) - It is very efficient in terms of storage, as it automatically deduplicates when possible (i.e. so that having two identical files does not consume more storage than having just one.) (The concept is similar to a "hard link", but subsequent modifications will not affect the original file.) - It preserves Mac-specific metadata to the maximum degree, including not only xattrs but also metadata not easily settable by other methods, including Finder and Spotlight params. When server-side "clone" is not available (for example, on non-APFS volumes), it falls back to server-side "copy" (still preserving metadata but using more disk storage.) It is only used when both remotes are local (and not wrapped by other remotes, such as crypt.) The behavior of local on non-mac systems is unchanged. --- backend/local/clone_darwin.go | 66 ++++++++++++++++++++++++++++ backend/union/union_internal_test.go | 7 +++ fs/operations/copy_test.go | 9 ++++ fs/sync/sync_test.go | 14 ++++++ go.mod | 1 + go.sum | 2 + 6 files changed, 99 insertions(+) create mode 100644 backend/local/clone_darwin.go diff --git a/backend/local/clone_darwin.go b/backend/local/clone_darwin.go new file mode 100644 index 000000000..2dbff2c05 --- /dev/null +++ b/backend/local/clone_darwin.go @@ -0,0 +1,66 @@ +//go:build darwin && cgo + +// Package local provides a filesystem interface +package local + +import ( + "context" + "runtime" + + "github.com/go-darwin/apfs" + "github.com/rclone/rclone/fs" +) + +// Copy src to this remote using server-side copy operations. +// +// # This is stored with the remote path given +// +// # It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + if runtime.GOOS != "darwin" || f.opt.TranslateSymlinks { + return nil, fs.ErrorCantCopy + } + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't clone - not same remote type") + return nil, fs.ErrorCantCopy + } + + // Create destination + dstObj := f.newObject(remote) + err := dstObj.mkdirAll() + if err != nil { + return nil, err + } + + err = Clone(srcObj.path, f.localPath(remote)) + if err != nil { + return nil, err + } + fs.Debugf(remote, "server-side cloned!") + return f.NewObject(ctx, remote) +} + +// Clone uses APFS cloning if possible, otherwise falls back to copying (with full metadata preservation) +// note that this is closely related to unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW) but not 100% identical +// https://opensource.apple.com/source/copyfile/copyfile-173.40.2/copyfile.c.auto.html +func Clone(src, dst string) error { + state := apfs.CopyFileStateAlloc() + defer func() { + if err := apfs.CopyFileStateFree(state); err != nil { + fs.Errorf(dst, "free state error: %v", err) + } + }() + cloned, err := apfs.CopyFile(src, dst, state, apfs.COPYFILE_CLONE) + fs.Debugf(dst, "isCloned: %v, error: %v", cloned, err) + return err +} + +// Check the interfaces are satisfied +var ( + _ fs.Copier = &Fs{} +) diff --git a/backend/union/union_internal_test.go b/backend/union/union_internal_test.go index b6f78ec8d..ccc130e8f 100644 --- a/backend/union/union_internal_test.go +++ b/backend/union/union_internal_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "runtime" "testing" "time" @@ -95,6 +96,12 @@ func TestMoveCopy(t *testing.T) { fLocal := unionFs.upstreams[0].Fs fMemory := unionFs.upstreams[1].Fs + if runtime.GOOS == "darwin" { + // need to disable as this test specifically tests a local that can't Copy + f.Features().Disable("Copy") + fLocal.Features().Disable("Copy") + } + t.Run("Features", func(t *testing.T) { assert.NotNil(t, f.Features().Move) assert.Nil(t, f.Features().Copy) diff --git a/fs/operations/copy_test.go b/fs/operations/copy_test.go index 73aa5e05a..1e8f3ae5e 100644 --- a/fs/operations/copy_test.go +++ b/fs/operations/copy_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path" + "runtime" "sort" "strings" "testing" @@ -449,6 +450,14 @@ func TestCopyFileMaxTransfer(t *testing.T) { ci.MaxTransfer = sizeCutoff ci.CutoffMode = fs.CutoffModeHard + if runtime.GOOS == "darwin" { + // disable server-side copies as they don't count towards transfer size stats + r.Flocal.Features().Disable("Copy") + if r.Fremote.Features().IsLocal { + r.Fremote.Features().Disable("Copy") + } + } + // file1: Show a small file gets transferred OK accounting.Stats(ctx).ResetCounters() err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path) diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 2d221e492..f1c07921e 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -1382,6 +1382,12 @@ func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) { r.CheckLocalItems(t, file1, file2) r.CheckRemoteItems(t) + if runtime.GOOS == "darwin" { + r.Flocal.Features().Disable("Copy") // macOS cloning is too fast for this test! + if r.Fremote.Features().IsLocal { + r.Fremote.Features().Disable("Copy") // macOS cloning is too fast for this test! + } + } accounting.GlobalStats().ResetCounters() // ctx = predictDstFromLogger(ctx) // not currently supported (but tests do pass for CutoffModeSoft) startTime := time.Now() @@ -2569,6 +2575,14 @@ func TestMaxTransfer(t *testing.T) { r.CheckLocalItems(t, file1, file2, file3) r.CheckRemoteItems(t) + if runtime.GOOS == "darwin" { + // disable server-side copies as they don't count towards transfer size stats + r.Flocal.Features().Disable("Copy") + if r.Fremote.Features().IsLocal { + r.Fremote.Features().Disable("Copy") + } + } + accounting.GlobalStats().ResetCounters() // ctx = predictDstFromLogger(ctx) // not currently supported diff --git a/go.mod b/go.mod index a29a24326..26f74b251 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/gabriel-vasile/mimetype v1.4.4 github.com/gdamore/tcell/v2 v2.7.4 github.com/go-chi/chi/v5 v5.1.0 + github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 github.com/go-git/go-billy/v5 v5.5.0 github.com/google/uuid v1.6.0 github.com/hanwen/go-fuse/v2 v2.5.1 diff --git a/go.sum b/go.sum index 1ade31d23..174178cd5 100644 --- a/go.sum +++ b/go.sum @@ -225,6 +225,8 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68= +github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=