From 205667143c807103b493e94bc0a708e10c435598 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 6 Mar 2025 16:45:52 +0000 Subject: [PATCH] serve nfs: make metadata files have special file handles Metadata files have the file handle of their source file with 0x00000001 suffixed in big endian so we can look them up directly from their file handles. --- cmd/nfsmount/nfsmount_test.go | 4 +- cmd/serve/nfs/cache.go | 78 ++++++++++++++++++++++++++-- cmd/serve/nfs/cache_test.go | 25 ++++++++- cmd/serve/nfs/nfs.go | 6 +++ cmd/serve/nfs/symlink_cache_linux.go | 13 +++++ 5 files changed, 120 insertions(+), 6 deletions(-) diff --git a/cmd/nfsmount/nfsmount_test.go b/cmd/nfsmount/nfsmount_test.go index ba6f350dc..a8402e901 100644 --- a/cmd/nfsmount/nfsmount_test.go +++ b/cmd/nfsmount/nfsmount_test.go @@ -11,6 +11,8 @@ import ( "testing" "github.com/rclone/rclone/cmd/serve/nfs" + "github.com/rclone/rclone/fs/object" + "github.com/rclone/rclone/vfs" "github.com/rclone/rclone/vfs/vfscommon" "github.com/rclone/rclone/vfs/vfstest" "github.com/stretchr/testify/require" @@ -38,7 +40,7 @@ func TestMount(t *testing.T) { nfs.Opt.HandleCacheDir = t.TempDir() require.NoError(t, nfs.Opt.HandleCache.Set(cacheType)) // Check we can create a handler - _, err := nfs.NewHandler(context.Background(), nil, &nfs.Opt) + _, err := nfs.NewHandler(context.Background(), vfs.New(object.MemoryFs, nil), &nfs.Opt) if errors.Is(err, nfs.ErrorSymlinkCacheNotSupported) || errors.Is(err, nfs.ErrorSymlinkCacheNoPermission) { t.Skip(err.Error() + ": run with: go test -c && sudo setcap cap_dac_read_search+ep ./nfsmount.test && ./nfsmount.test -test.v") } diff --git a/cmd/serve/nfs/cache.go b/cmd/serve/nfs/cache.go index 32c6cc16c..278d923dc 100644 --- a/cmd/serve/nfs/cache.go +++ b/cmd/serve/nfs/cache.go @@ -3,6 +3,7 @@ package nfs import ( + "bytes" "crypto/md5" "encoding/hex" "errors" @@ -30,6 +31,15 @@ var ( ErrorSymlinkCacheNoPermission = errors.New("symlink cache must be run as root or with CAP_DAC_READ_SEARCH") ) +// Metadata files have the file handle of their source file with this +// suffixed so we can look them up directly from the file handle. +// +// Note that this is 4 bytes - using a non multiple of 4 will cause +// the Linux NFS client not to be able to read any files. +// +// The value is big endian 0x00000001 +var metadataSuffix = []byte{0x00, 0x00, 0x00, 0x01} + // Cache controls the file handle cache implementation type Cache interface { // ToHandle takes a file and represents it with an opaque handle to reference it. @@ -77,7 +87,9 @@ type diskHandler struct { write func(fh []byte, cachePath string, fullPath string) ([]byte, error) read func(fh []byte, cachePath string) ([]byte, error) remove func(fh []byte, cachePath string) error - handleType int32 //nolint:unused // used by the symlink cache + suffix func(fh []byte) []byte // returns nil for no suffix or the suffix + handleType int32 //nolint:unused // used by the symlink cache + metadata string // extension for metadata } // Create a new disk handler @@ -102,6 +114,8 @@ func newDiskHandler(h *Handler) (dh *diskHandler, err error) { write: dh.diskCacheWrite, read: dh.diskCacheRead, remove: dh.diskCacheRemove, + suffix: dh.diskCacheSuffix, + metadata: h.vfs.Opt.MetadataExtension, } fs.Infof("nfs", "Storing handle cache in %q", dh.cacheDir) return dh, nil @@ -124,6 +138,17 @@ func (dh *diskHandler) handleToPath(fh []byte) (cachePath string) { return cachePath } +// Return true if name represents a metadata file +// +// It returns the underlying path +func (dh *diskHandler) isMetadataFile(name string) (rawName string, found bool) { + if dh.metadata == "" { + return name, false + } + rawName, found = strings.CutSuffix(name, dh.metadata) + return rawName, found +} + // ToHandle takes a file and represents it with an opaque handle to reference it. // In stateless nfs (when it's serving a unix fs) this can be the device + inode // but we can generalize with a stateful local cache of handed out IDs. @@ -131,6 +156,8 @@ func (dh *diskHandler) ToHandle(f billy.Filesystem, splitPath []string) (fh []by dh.mu.Lock() defer dh.mu.Unlock() fullPath := path.Join(splitPath...) + // metadata file has file handle of original file + fullPath, isMetadataFile := dh.isMetadataFile(fullPath) fh = hashPath(fullPath) cachePath := dh.handleToPath(fh) cacheDir := filepath.Dir(cachePath) @@ -144,6 +171,10 @@ func (dh *diskHandler) ToHandle(f billy.Filesystem, splitPath []string) (fh []by fs.Errorf("nfs", "Couldn't create cache file handle: %v", err) return fh } + // metadata file handle is suffixed with metadataSuffix + if isMetadataFile { + fh = append(fh, metadataSuffix...) + } return fh } @@ -152,18 +183,43 @@ func (dh *diskHandler) diskCacheWrite(fh []byte, cachePath string, fullPath stri return fh, os.WriteFile(cachePath, []byte(fullPath), 0600) } -var errStaleHandle = &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale} +var ( + errStaleHandle = &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale} +) + +// Test to see if a fh is a metadata handle and if so return the underlying handle +func (dh *diskHandler) isMetadataHandle(fh []byte) (isMetadata bool, newFh []byte, err error) { + if dh.metadata == "" { + return false, fh, nil + } + suffix := dh.suffix(fh) + if len(suffix) == 0 { + // OK + return false, fh, nil + } else if bytes.Equal(suffix, metadataSuffix) { + return true, fh[:len(fh)-len(suffix)], nil + } + fs.Errorf("nfs", "Bad file handle suffix %X", suffix) + return false, nil, errStaleHandle +} // FromHandle converts from an opaque handle to the file it represents func (dh *diskHandler) FromHandle(fh []byte) (f billy.Filesystem, splitPath []string, err error) { dh.mu.RLock() defer dh.mu.RUnlock() + isMetadata, fh, err := dh.isMetadataHandle(fh) + if err != nil { + return nil, nil, err + } cachePath := dh.handleToPath(fh) fullPathBytes, err := dh.read(fh, cachePath) if err != nil { fs.Errorf("nfs", "Stale handle %q: %v", cachePath, err) return nil, nil, errStaleHandle } + if isMetadata { + fullPathBytes = append(fullPathBytes, []byte(dh.metadata)...) + } splitPath = strings.Split(string(fullPathBytes), "/") return dh.billyFS, splitPath, nil } @@ -177,8 +233,16 @@ func (dh *diskHandler) diskCacheRead(fh []byte, cachePath string) ([]byte, error func (dh *diskHandler) InvalidateHandle(f billy.Filesystem, fh []byte) error { dh.mu.Lock() defer dh.mu.Unlock() + isMetadata, fh, err := dh.isMetadataHandle(fh) + if err != nil { + return err + } + if isMetadata { + // Can't invalidate a metadata handle as it is synthetic + return nil + } cachePath := dh.handleToPath(fh) - err := dh.remove(fh, cachePath) + err = dh.remove(fh, cachePath) if err != nil { fs.Errorf("nfs", "Failed to remove handle %q: %v", cachePath, err) } @@ -190,6 +254,14 @@ func (dh *diskHandler) diskCacheRemove(fh []byte, cachePath string) error { return os.Remove(cachePath) } +// Return a suffix for the file handle or nil +func (dh *diskHandler) diskCacheSuffix(fh []byte) []byte { + if len(fh) <= md5.Size { + return nil + } + return fh[md5.Size:] +} + // HandleLimit exports how many file handles can be safely stored by this cache. func (dh *diskHandler) HandleLimit() int { return math.MaxInt diff --git a/cmd/serve/nfs/cache_test.go b/cmd/serve/nfs/cache_test.go index 03100d34d..6941fde17 100644 --- a/cmd/serve/nfs/cache_test.go +++ b/cmd/serve/nfs/cache_test.go @@ -5,10 +5,13 @@ package nfs import ( "context" "fmt" + "strings" "sync" "testing" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/object" + "github.com/rclone/rclone/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -18,6 +21,8 @@ const testSymlinkCache = "go test -c && sudo setcap cap_dac_read_search+ep ./nfs // Check basic CRUD operations func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) { + isMetadata := strings.HasSuffix(fileName, ".metadata") + // Check reading a non existent handle returns an error _, _, err := c.FromHandle([]byte{10}) assert.Error(t, err) @@ -26,6 +31,11 @@ func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) { splitPath := []string{"dir", fileName} fh := c.ToHandle(h.billyFS, splitPath) assert.True(t, len(fh) > 0) + if isMetadata { + assert.Equal(t, metadataSuffix, fh[len(fh)-len(metadataSuffix):]) + } else { + assert.NotEqual(t, metadataSuffix, fh[len(fh)-len(metadataSuffix):]) + } // Read the handle back newFs, newSplitPath, err := c.FromHandle(fh) @@ -43,8 +53,13 @@ func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) { // Check the handle is gone and returning stale handle error _, _, err = c.FromHandle(fh) - require.Error(t, err) - assert.Equal(t, errStaleHandle, err) + if !isMetadata { + require.Error(t, err) + assert.Equal(t, errStaleHandle, err) + } else { + // Can't invalidate metadata handles + require.NoError(t, err) + } } // Thrash the cache operations in parallel on different files @@ -113,8 +128,10 @@ func TestCache(t *testing.T) { cacheType := cacheType t.Run(cacheType.String(), func(t *testing.T) { h := &Handler{ + vfs: vfs.New(object.MemoryFs, nil), billyFS: billyFS, } + h.vfs.Opt.MetadataExtension = ".metadata" h.opt.HandleLimit = 1000 h.opt.HandleCache = cacheType h.opt.HandleCacheDir = t.TempDir() @@ -151,6 +168,10 @@ func TestCache(t *testing.T) { t.Run("ThrashSame", func(t *testing.T) { testCacheThrashSame(t, h, c) }) + // Metadata file handles only supported on non memory + t.Run("CRUDMetadata", func(t *testing.T) { + testCacheCRUD(t, h, c, "file.metadata") + }) } }) } diff --git a/cmd/serve/nfs/nfs.go b/cmd/serve/nfs/nfs.go index ad94c5a49..6e6878e3c 100644 --- a/cmd/serve/nfs/nfs.go +++ b/cmd/serve/nfs/nfs.go @@ -169,6 +169,12 @@ Where |$PORT| is the same port number used in the |serve nfs| command and |$HOSTNAME| is the network address of the machine that |serve nfs| was run on. +If |--vfs-metadata-extension| is in use then for the |--nfs-cache-type disk| +and |--nfs-cache-type cache| the metadata files will have the file +handle of their parent file suffixed with |0x00, 0x00, 0x00, 0x01|. +This means they can be looked up directly from the parent file handle +is desired. + This command is only available on Unix platforms. `, "|", "`") + vfs.Help(), diff --git a/cmd/serve/nfs/symlink_cache_linux.go b/cmd/serve/nfs/symlink_cache_linux.go index 8002824ff..806de6685 100644 --- a/cmd/serve/nfs/symlink_cache_linux.go +++ b/cmd/serve/nfs/symlink_cache_linux.go @@ -82,6 +82,7 @@ func (dh *diskHandler) makeSymlinkCache() error { dh.read = dh.symlinkCacheRead dh.write = dh.symlinkCacheWrite dh.remove = dh.symlinkCacheRemove + dh.suffix = dh.symlinkCacheSuffix return nil } @@ -208,3 +209,15 @@ func (dh *diskHandler) symlinkCacheRemove(fh []byte, cachePath string) error { return os.Remove(cachePath) } + +// Return a suffix for the file handle or nil +func (dh *diskHandler) symlinkCacheSuffix(fh []byte) []byte { + if len(fh) < 4 { + return nil + } + length := int(binary.BigEndian.Uint32(fh[:4])) + 4 + if len(fh) <= length { + return nil + } + return fh[length:] +}