serve nfs: make metadata files have special file handles

Metadata files have the file handle of their source file with
0x00000001 suffixed in big endian so we can look them up directly from
their file handles.

cmd/nfsmount/nfsmount_test.go

@@ -11,6 +11,8 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/cmd/serve/nfs"
+	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/rclone/rclone/vfs/vfstest"
 	"github.com/stretchr/testify/require"
@@ -38,7 +40,7 @@ func TestMount(t *testing.T) {
 			nfs.Opt.HandleCacheDir = t.TempDir()
 			require.NoError(t, nfs.Opt.HandleCache.Set(cacheType))
 			// Check we can create a handler
-			_, err := nfs.NewHandler(context.Background(), nil, &nfs.Opt)
+			_, err := nfs.NewHandler(context.Background(), vfs.New(object.MemoryFs, nil), &nfs.Opt)
 			if errors.Is(err, nfs.ErrorSymlinkCacheNotSupported) || errors.Is(err, nfs.ErrorSymlinkCacheNoPermission) {
 				t.Skip(err.Error() + ": run with: go test -c && sudo setcap cap_dac_read_search+ep ./nfsmount.test && ./nfsmount.test -test.v")
 			}

cmd/serve/nfs/cache.go

@@ -3,6 +3,7 @@
 package nfs
 
 import (
+	"bytes"
 	"crypto/md5"
 	"encoding/hex"
 	"errors"
@@ -30,6 +31,15 @@ var (
 	ErrorSymlinkCacheNoPermission = errors.New("symlink cache must be run as root or with CAP_DAC_READ_SEARCH")
 )
 
+// Metadata files have the file handle of their source file with this
+// suffixed so we can look them up directly from the file handle.
+//
+// Note that this is 4 bytes - using a non multiple of 4 will cause
+// the Linux NFS client not to be able to read any files.
+//
+// The value is big endian 0x00000001
+var metadataSuffix = []byte{0x00, 0x00, 0x00, 0x01}
+
 // Cache controls the file handle cache implementation
 type Cache interface {
 	// ToHandle takes a file and represents it with an opaque handle to reference it.
@@ -77,7 +87,9 @@ type diskHandler struct {
 	write      func(fh []byte, cachePath string, fullPath string) ([]byte, error)
 	read       func(fh []byte, cachePath string) ([]byte, error)
 	remove     func(fh []byte, cachePath string) error
-	handleType int32 //nolint:unused // used by the symlink cache
+	suffix     func(fh []byte) []byte // returns nil for no suffix or the suffix
+	handleType int32                  //nolint:unused // used by the symlink cache
+	metadata   string                 // extension for metadata
 }
 
 // Create a new disk handler
@@ -102,6 +114,8 @@ func newDiskHandler(h *Handler) (dh *diskHandler, err error) {
 		write:    dh.diskCacheWrite,
 		read:     dh.diskCacheRead,
 		remove:   dh.diskCacheRemove,
+		suffix:   dh.diskCacheSuffix,
+		metadata: h.vfs.Opt.MetadataExtension,
 	}
 	fs.Infof("nfs", "Storing handle cache in %q", dh.cacheDir)
 	return dh, nil
@@ -124,6 +138,17 @@ func (dh *diskHandler) handleToPath(fh []byte) (cachePath string) {
 	return cachePath
 }
 
+// Return true if name represents a metadata file
+//
+// It returns the underlying path
+func (dh *diskHandler) isMetadataFile(name string) (rawName string, found bool) {
+	if dh.metadata == "" {
+		return name, false
+	}
+	rawName, found = strings.CutSuffix(name, dh.metadata)
+	return rawName, found
+}
+
 // ToHandle takes a file and represents it with an opaque handle to reference it.
 // In stateless nfs (when it's serving a unix fs) this can be the device + inode
 // but we can generalize with a stateful local cache of handed out IDs.
@@ -131,6 +156,8 @@ func (dh *diskHandler) ToHandle(f billy.Filesystem, splitPath []string) (fh []by
 	dh.mu.Lock()
 	defer dh.mu.Unlock()
 	fullPath := path.Join(splitPath...)
+	// metadata file has file handle of original file
+	fullPath, isMetadataFile := dh.isMetadataFile(fullPath)
 	fh = hashPath(fullPath)
 	cachePath := dh.handleToPath(fh)
 	cacheDir := filepath.Dir(cachePath)
@@ -144,6 +171,10 @@ func (dh *diskHandler) ToHandle(f billy.Filesystem, splitPath []string) (fh []by
 		fs.Errorf("nfs", "Couldn't create cache file handle: %v", err)
 		return fh
 	}
+	// metadata file handle is suffixed with metadataSuffix
+	if isMetadataFile {
+		fh = append(fh, metadataSuffix...)
+	}
 	return fh
 }
 
@@ -152,18 +183,43 @@ func (dh *diskHandler) diskCacheWrite(fh []byte, cachePath string, fullPath stri
 	return fh, os.WriteFile(cachePath, []byte(fullPath), 0600)
 }
 
-var errStaleHandle = &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale}
+var (
+	errStaleHandle = &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale}
+)
+
+// Test to see if a fh is a metadata handle and if so return the underlying handle
+func (dh *diskHandler) isMetadataHandle(fh []byte) (isMetadata bool, newFh []byte, err error) {
+	if dh.metadata == "" {
+		return false, fh, nil
+	}
+	suffix := dh.suffix(fh)
+	if len(suffix) == 0 {
+		// OK
+		return false, fh, nil
+	} else if bytes.Equal(suffix, metadataSuffix) {
+		return true, fh[:len(fh)-len(suffix)], nil
+	}
+	fs.Errorf("nfs", "Bad file handle suffix %X", suffix)
+	return false, nil, errStaleHandle
+}
 
 // FromHandle converts from an opaque handle to the file it represents
 func (dh *diskHandler) FromHandle(fh []byte) (f billy.Filesystem, splitPath []string, err error) {
 	dh.mu.RLock()
 	defer dh.mu.RUnlock()
+	isMetadata, fh, err := dh.isMetadataHandle(fh)
+	if err != nil {
+		return nil, nil, err
+	}
 	cachePath := dh.handleToPath(fh)
 	fullPathBytes, err := dh.read(fh, cachePath)
 	if err != nil {
 		fs.Errorf("nfs", "Stale handle %q: %v", cachePath, err)
 		return nil, nil, errStaleHandle
 	}
+	if isMetadata {
+		fullPathBytes = append(fullPathBytes, []byte(dh.metadata)...)
+	}
 	splitPath = strings.Split(string(fullPathBytes), "/")
 	return dh.billyFS, splitPath, nil
 }
@@ -177,8 +233,16 @@ func (dh *diskHandler) diskCacheRead(fh []byte, cachePath string) ([]byte, error
 func (dh *diskHandler) InvalidateHandle(f billy.Filesystem, fh []byte) error {
 	dh.mu.Lock()
 	defer dh.mu.Unlock()
+	isMetadata, fh, err := dh.isMetadataHandle(fh)
+	if err != nil {
+		return err
+	}
+	if isMetadata {
+		// Can't invalidate a metadata handle as it is synthetic
+		return nil
+	}
 	cachePath := dh.handleToPath(fh)
-	err := dh.remove(fh, cachePath)
+	err = dh.remove(fh, cachePath)
 	if err != nil {
 		fs.Errorf("nfs", "Failed to remove handle %q: %v", cachePath, err)
 	}
@@ -190,6 +254,14 @@ func (dh *diskHandler) diskCacheRemove(fh []byte, cachePath string) error {
 	return os.Remove(cachePath)
 }
 
+// Return a suffix for the file handle or nil
+func (dh *diskHandler) diskCacheSuffix(fh []byte) []byte {
+	if len(fh) <= md5.Size {
+		return nil
+	}
+	return fh[md5.Size:]
+}
+
 // HandleLimit exports how many file handles can be safely stored by this cache.
 func (dh *diskHandler) HandleLimit() int {
 	return math.MaxInt

cmd/serve/nfs/cache_test.go

@@ -5,10 +5,13 @@ package nfs
 import (
 	"context"
 	"fmt"
+	"strings"
 	"sync"
 	"testing"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/vfs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -18,6 +21,8 @@ const testSymlinkCache = "go test -c && sudo setcap cap_dac_read_search+ep ./nfs
 
 // Check basic CRUD operations
 func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) {
+	isMetadata := strings.HasSuffix(fileName, ".metadata")
+
 	// Check reading a non existent handle returns an error
 	_, _, err := c.FromHandle([]byte{10})
 	assert.Error(t, err)
@@ -26,6 +31,11 @@ func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) {
 	splitPath := []string{"dir", fileName}
 	fh := c.ToHandle(h.billyFS, splitPath)
 	assert.True(t, len(fh) > 0)
+	if isMetadata {
+		assert.Equal(t, metadataSuffix, fh[len(fh)-len(metadataSuffix):])
+	} else {
+		assert.NotEqual(t, metadataSuffix, fh[len(fh)-len(metadataSuffix):])
+	}
 
 	// Read the handle back
 	newFs, newSplitPath, err := c.FromHandle(fh)
@@ -43,8 +53,13 @@ func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) {
 
 	// Check the handle is gone and returning stale handle error
 	_, _, err = c.FromHandle(fh)
-	require.Error(t, err)
+	if !isMetadata {
-	assert.Equal(t, errStaleHandle, err)
+		require.Error(t, err)
+		assert.Equal(t, errStaleHandle, err)
+	} else {
+		// Can't invalidate metadata handles
+		require.NoError(t, err)
+	}
 }
 
 // Thrash the cache operations in parallel on different files
@@ -113,8 +128,10 @@ func TestCache(t *testing.T) {
 		cacheType := cacheType
 		t.Run(cacheType.String(), func(t *testing.T) {
 			h := &Handler{
+				vfs:     vfs.New(object.MemoryFs, nil),
 				billyFS: billyFS,
 			}
+			h.vfs.Opt.MetadataExtension = ".metadata"
 			h.opt.HandleLimit = 1000
 			h.opt.HandleCache = cacheType
 			h.opt.HandleCacheDir = t.TempDir()
@@ -151,6 +168,10 @@ func TestCache(t *testing.T) {
 				t.Run("ThrashSame", func(t *testing.T) {
 					testCacheThrashSame(t, h, c)
 				})
+				// Metadata file handles only supported on non memory
+				t.Run("CRUDMetadata", func(t *testing.T) {
+					testCacheCRUD(t, h, c, "file.metadata")
+				})
 			}
 		})
 	}

cmd/serve/nfs/nfs.go

@@ -169,6 +169,12 @@ Where |$PORT| is the same port number used in the |serve nfs| command
 and |$HOSTNAME| is the network address of the machine that |serve nfs|
 was run on.
 
+If |--vfs-metadata-extension| is in use then for the |--nfs-cache-type disk|
+and |--nfs-cache-type cache| the metadata files will have the file
+handle of their parent file suffixed with |0x00, 0x00, 0x00, 0x01|.
+This means they can be looked up directly from the parent file handle
+is desired.
+
 This command is only available on Unix platforms.
 
 `, "|", "`") + vfs.Help(),

cmd/serve/nfs/symlink_cache_linux.go

@@ -82,6 +82,7 @@ func (dh *diskHandler) makeSymlinkCache() error {
 	dh.read = dh.symlinkCacheRead
 	dh.write = dh.symlinkCacheWrite
 	dh.remove = dh.symlinkCacheRemove
+	dh.suffix = dh.symlinkCacheSuffix
 
 	return nil
 }
@@ -208,3 +209,15 @@ func (dh *diskHandler) symlinkCacheRemove(fh []byte, cachePath string) error {
 
 	return os.Remove(cachePath)
 }
+
+// Return a suffix for the file handle or nil
+func (dh *diskHandler) symlinkCacheSuffix(fh []byte) []byte {
+	if len(fh) < 4 {
+		return nil
+	}
+	length := int(binary.BigEndian.Uint32(fh[:4])) + 4
+	if len(fh) <= length {
+		return nil
+	}
+	return fh[length:]
+}