cmd/gitannex: Port unit tests to fstest

This enables the unit tests to run on any given backend, via the
`-remote` flag, e.g. `go test -v ./cmd/gitannex/... -remote dropbox:`.

We should also port the gitannex e2e tests at some point.

Issue: #7984

cmd/gitannex/gitannex.go

@@ -7,7 +7,7 @@
 // (Tracked in [issue #7625].)
 //
 //  1. ✅ Minimal support for the [external special remote protocol]. Tested on
-//     "local" and "drive" backends.
+//     "local", "drive", and "dropbox" backends.
 //  2. Add support for the ASYNC protocol extension. This may improve performance.
 //  3. Support the [simple export interface]. This will enable `git-annex
 //     export` functionality.
@@ -33,6 +33,7 @@ import (
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/spf13/cobra"
 )
@@ -267,21 +268,33 @@ func (s *server) handleInitRemote() error {
 		return fmt.Errorf("failed to get configs: %w", err)
 	}
 
-	remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName))
+	// Explicitly check that a remote with the given name exists. If we just
-	if err != nil {
+	// relied on `cache.Get()` to return `fs.ErrorNotFoundInConfigFile`, this
-		s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote")
+	// function would incorrectly succeed when the given remote name is actually
-		return fmt.Errorf("failed to open root directory of rclone remote: %w", err)
+	// a file path.
+	//
+	// The :local: backend does not correspond to a remote named by the rclone
+	// config, but is permitted to enable testing. Technically, a user might hit
+	// this code path, but it would be a strange choice because git-annex
+	// natively supports a "directory" special remote.
+
+	trimmedName := strings.TrimSuffix(s.configRcloneRemoteName, ":")
+
+	if s.configRcloneRemoteName != ":local" {
+		var remoteExists bool
+		for _, remoteName := range config.FileSections() {
+			if remoteName == trimmedName {
+				remoteExists = true
+				break
+			}
+		}
+		if !remoteExists {
+			s.sendMsg("INITREMOTE-FAILURE remote does not exist: " + s.configRcloneRemoteName)
+			return fmt.Errorf("remote does not exist: %s", s.configRcloneRemoteName)
+		}
 	}
 
-	if !remoteRootFs.Features().CanHaveEmptyDirectories {
+	s.configRcloneRemoteName = trimmedName + ":"
-		s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories")
-		return fmt.Errorf("rclone remote does not support empty directories")
-	}
-
-	if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil {
-		s.sendMsg("INITREMOTE-FAILURE failed to mkdir")
-		return fmt.Errorf("failed to mkdir: %w", err)
-	}
 
 	s.sendMsg("INITREMOTE-SUCCESS")
 	return nil

cmd/gitannex/gitannex_test.go

@@ -2,25 +2,24 @@ package gitannex
 
 import (
 	"bufio"
-	"crypto/sha256"
+	"context"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
-	// Without this import, the local filesystem backend would be unavailable.
+	// Without this import, the various backends would be unavailable. It looks
-	// It looks unused, but the act of importing it runs its `init()` function.
+	// unused, but the act of importing runs the package's `init()` function.
-	_ "github.com/rclone/rclone/backend/local"
+	_ "github.com/rclone/rclone/backend/all"
 
-	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fspath"
-	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configfile"
-	"github.com/rclone/rclone/fstest/mockfs"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -256,9 +255,9 @@ type testState struct {
 	mockStdinW       *io.PipeWriter
 	mockStdoutReader *bufio.Reader
 
-	localFsDir string
+	fstestRun    *fstest.Run
-	configPath string
+	remoteName   string
-	remoteName string
+	remotePrefix string
 }
 
 func makeTestState(t *testing.T) testState {
@@ -276,6 +275,10 @@ func makeTestState(t *testing.T) testState {
 	}
 }
 
+func (h *testState) requireRemoteIsEmpty() {
+	h.fstestRun.CheckRemoteItems(h.t)
+}
+
 func (h *testState) requireReadLineExact(line string) {
 	receivedLine, err := h.mockStdoutReader.ReadString('\n')
 	require.NoError(h.t, err)
@@ -296,21 +299,106 @@ func (h *testState) requireWriteLine(line string) {
 // Preconfigure the handle. This enables the calling test to skip the PREPARE
 // handshake.
 func (h *testState) preconfigureServer() {
-	h.server.configPrefix = h.localFsDir
 	h.server.configRcloneRemoteName = h.remoteName
+	h.server.configPrefix = h.remotePrefix
 	h.server.configRcloneLayout = string(layoutModeNodir)
 	h.server.configsDone = true
 }
 
-// getUniqueRemoteName returns a valid remote name derived from the given test's
+// Drop-in replacement for `filepath.Rel()` that works around a Windows-specific
-// name. This is necessary because when a test registers a second remote with
+// quirk when one of the paths begins with `\\?\` or `//?/`. It seems that
-// the same name, the original remote appears to take precedence. This function
+// fstest gives us paths with this prefix on Windows, which throws a wrench in
-// is injective, so each test gets a unique remote name. Returned strings
+// the gitannex tests that need to construct relative paths from absolute paths.
-// contain no spaces.
+// For a demonstration, see `TestWindowsFilepathRelQuirk` below.
-func getUniqueRemoteName(t *testing.T) string {
+//
-	// Using sha256 as a hack to ensure injectivity without adding a global
+// The `\\?\` prefix tells Windows APIs to pass strings unmodified to the
-	// variable.
+// filesystem without additional parsing [1]. Our workaround is roughly to add
-	return fmt.Sprintf("remote-%x", sha256.Sum256([]byte(t.Name())))
+// the prefix to whichever parameter doesn't have it (when the OS is Windows).
+// I'm not sure this generalizes, but it works for the the kinds of inputs we're
+// throwing at it.
+//
+// [1]: https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file?redirectedfrom=MSDN#win32-file-namespaces
+func relativeFilepathWorkaround(basepath, targpath string) (string, error) {
+	if runtime.GOOS != "windows" {
+		return filepath.Rel(basepath, targpath)
+	}
+	// Canonicalize paths to use backslashes.
+	basepath = filepath.Clean(basepath)
+	targpath = filepath.Clean(targpath)
+
+	const winFilePrefixDisableStringParsing = `\\?\`
+	baseHasPrefix := strings.HasPrefix(basepath, winFilePrefixDisableStringParsing)
+	targHasPrefix := strings.HasPrefix(targpath, winFilePrefixDisableStringParsing)
+
+	if baseHasPrefix && !targHasPrefix {
+		targpath = winFilePrefixDisableStringParsing + targpath
+	}
+	if !baseHasPrefix && targHasPrefix {
+		basepath = winFilePrefixDisableStringParsing + basepath
+	}
+	return filepath.Rel(basepath, targpath)
+}
+
+func TestWindowsFilepathRelQuirk(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		t.Skip()
+	}
+
+	t.Run("filepathRelQuirk", func(t *testing.T) {
+		var err error
+
+		_, err = filepath.Rel(`C:\foo`, `\\?\C:\foo\bar`)
+		require.Error(t, err)
+
+		_, err = filepath.Rel(`C:/foo`, `//?/C:/foo/bar`)
+		require.Error(t, err)
+
+		_, err = filepath.Rel(`\\?\C:\foo`, `C:\foo\bar`)
+		require.Error(t, err)
+
+		_, err = filepath.Rel(`//?/C:/foo`, `C:/foo/bar`)
+		require.Error(t, err)
+
+		path, err := filepath.Rel(`\\?\C:\foo`, `\\?\C:\foo\bar`)
+		require.NoError(t, err)
+		require.Equal(t, path, `bar`)
+
+		path, err = filepath.Rel(`//?/C:/foo`, `//?/C:/foo/bar`)
+		require.NoError(t, err)
+		require.Equal(t, path, `bar`)
+	})
+
+	t.Run("fstestAndTempDirHaveDifferentPrefixes", func(t *testing.T) {
+		r := fstest.NewRun(t)
+		p := r.Flocal.Root()
+		require.True(t, strings.HasPrefix(p, `//?/`))
+
+		tempDir := t.TempDir()
+		require.False(t, strings.HasPrefix(tempDir, `//?/`))
+		require.False(t, strings.HasPrefix(tempDir, `\\?\`))
+	})
+
+	t.Run("workaroundWorks", func(t *testing.T) {
+		path, err := relativeFilepathWorkaround(`C:\foo`, `\\?\C:\foo\bar`)
+		require.NoError(t, err)
+		require.Equal(t, path, "bar")
+
+		path, err = relativeFilepathWorkaround(`C:/foo`, `//?/C:/foo/bar`)
+		require.NoError(t, err)
+		require.Equal(t, path, "bar")
+
+		path, err = relativeFilepathWorkaround(`\\?\C:\foo`, `C:\foo\bar`)
+		require.NoError(t, err)
+		require.Equal(t, path, `bar`)
+
+		path, err = relativeFilepathWorkaround(`//?/C:/foo`, `C:/foo/bar`)
+		require.NoError(t, err)
+		require.Equal(t, path, `bar`)
+
+		path, err = relativeFilepathWorkaround(`\\?\C:\foo`, `\\?\C:\foo\bar`)
+		require.NoError(t, err)
+		require.Equal(t, path, `bar`)
+	})
 }
 
 type testCase struct {
@@ -319,8 +407,8 @@ type testCase struct {
 	expectedError    string
 }
 
-// These test cases run against the "local" backend.
+// These test cases run against a backend selected by the `-remote` flag.
-var localBackendTestCases = []testCase{
+var fstestTestCases = []testCase{
 	{
 		label: "HandlesInit",
 		testProtocolFunc: func(t *testing.T, h *testState) {
@@ -377,18 +465,86 @@ var localBackendTestCases = []testCase{
 			h.requireReadLineExact("GETCONFIG rcloneremotename")
 			h.requireWriteLine("VALUE " + h.remoteName)
 			h.requireReadLineExact("GETCONFIG rcloneprefix")
-			h.requireWriteLine("VALUE " + h.localFsDir)
+			h.requireWriteLine("VALUE " + h.remotePrefix)
 			h.requireReadLineExact("GETCONFIG rclonelayout")
 			h.requireWriteLine("VALUE foo")
 			h.requireReadLineExact("PREPARE-SUCCESS")
 
 			require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
-			require.Equal(t, h.server.configPrefix, h.localFsDir)
+			require.Equal(t, h.server.configPrefix, h.remotePrefix)
 			require.True(t, h.server.configsDone)
 
 			require.NoError(t, h.mockStdinW.Close())
 		},
 	},
+	{
+		label: "HandlesPrepareWithNonexistentRemote",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.requireReadLineExact("VERSION 1")
+			h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
+			h.requireReadLineExact("EXTENSIONS")
+
+			if !h.server.extensionInfo {
+				t.Errorf("expected INFO extension to be enabled")
+				return
+			}
+
+			h.requireWriteLine("PREPARE")
+			h.requireReadLineExact("GETCONFIG rcloneremotename")
+			h.requireWriteLine("VALUE thisRemoteDoesNotExist")
+			h.requireReadLineExact("GETCONFIG rcloneprefix")
+			h.requireWriteLine("VALUE " + h.remotePrefix)
+			h.requireReadLineExact("GETCONFIG rclonelayout")
+			h.requireWriteLine("VALUE foo")
+			h.requireReadLineExact("PREPARE-SUCCESS")
+
+			require.Equal(t, h.server.configRcloneRemoteName, "thisRemoteDoesNotExist")
+			require.Equal(t, h.server.configPrefix, h.remotePrefix)
+			require.True(t, h.server.configsDone)
+
+			h.requireWriteLine("INITREMOTE")
+			h.requireReadLineExact("INITREMOTE-FAILURE remote does not exist: thisRemoteDoesNotExist")
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+		expectedError: "remote does not exist: thisRemoteDoesNotExist",
+	},
+	{
+		label: "HandlesPrepareWithPathAsRemote",
+		testProtocolFunc: func(t *testing.T, h *testState) {
+			h.requireReadLineExact("VERSION 1")
+			h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
+			h.requireReadLineExact("EXTENSIONS")
+
+			if !h.server.extensionInfo {
+				t.Errorf("expected INFO extension to be enabled")
+				return
+			}
+
+			h.requireWriteLine("PREPARE")
+			h.requireReadLineExact("GETCONFIG rcloneremotename")
+			h.requireWriteLine("VALUE " + h.remotePrefix)
+			h.requireReadLineExact("GETCONFIG rcloneprefix")
+			h.requireWriteLine("VALUE /foo")
+			h.requireReadLineExact("GETCONFIG rclonelayout")
+			h.requireWriteLine("VALUE foo")
+			h.requireReadLineExact("PREPARE-SUCCESS")
+
+			require.Equal(t, h.server.configRcloneRemoteName, h.remotePrefix)
+			require.Equal(t, h.server.configPrefix, "/foo")
+			require.True(t, h.server.configsDone)
+
+			h.requireWriteLine("INITREMOTE")
+
+			require.Regexp(t,
+				regexp.MustCompile("^INITREMOTE-FAILURE remote does not exist: "),
+				h.requireReadLine(),
+			)
+
+			require.NoError(t, h.mockStdinW.Close())
+		},
+		expectedError: "remote does not exist:",
+	},
 	{
 		label: "HandlesPrepareWithSynonyms",
 		testProtocolFunc: func(t *testing.T, h *testState) {
@@ -409,13 +565,13 @@ var localBackendTestCases = []testCase{
 			h.requireWriteLine("VALUE " + h.remoteName)
 
 			h.requireReadLineExact("GETCONFIG rcloneprefix")
-			h.requireWriteLine("VALUE " + h.localFsDir)
+			h.requireWriteLine("VALUE " + h.remotePrefix)
 			h.requireReadLineExact("GETCONFIG rclonelayout")
 			h.requireWriteLine("VALUE foo")
 			h.requireReadLineExact("PREPARE-SUCCESS")
 
 			require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
-			require.Equal(t, h.server.configPrefix, h.localFsDir)
+			require.Equal(t, h.server.configPrefix, h.remotePrefix)
 			require.True(t, h.server.configsDone)
 
 			require.NoError(t, h.mockStdinW.Close())
@@ -437,12 +593,12 @@ var localBackendTestCases = []testCase{
 			h.requireReadLineExact("GETCONFIG rcloneremotename")
 
 			remoteNameWithSpaces := fmt.Sprintf(" %s ", h.remoteName)
-			localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)
+			prefixWithWhitespace := fmt.Sprintf(" %s\t", h.remotePrefix)
 
 			h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))
 
 			h.requireReadLineExact("GETCONFIG rcloneprefix")
-			h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))
+			h.requireWriteLine(fmt.Sprintf("VALUE %s", prefixWithWhitespace))
 
 			h.requireReadLineExact("GETCONFIG rclonelayout")
 			h.requireWriteLine("VALUE")
@@ -452,7 +608,7 @@ var localBackendTestCases = []testCase{
 			h.requireReadLineExact("PREPARE-SUCCESS")
 
 			require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)
-			require.Equal(t, h.server.configPrefix, localFsDirWithSpaces)
+			require.Equal(t, h.server.configPrefix, prefixWithWhitespace)
 			require.True(t, h.server.configsDone)
 
 			require.NoError(t, h.mockStdinW.Close())
@@ -639,20 +795,25 @@ var localBackendTestCases = []testCase{
 			h.requireReadLineExact("INITREMOTE-SUCCESS")
 
 			// Create temp file for transfer with an absolute path.
-			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
-			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+			absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
-			require.FileExists(t, fileToTransfer)
+			require.True(t, filepath.IsAbs(absPath))
-			require.True(t, filepath.IsAbs(fileToTransfer))
 
 			// Specify an absolute path to transfer.
-			h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
+			h.requireWriteLine("TRANSFER STORE KeyAbsolute " + absPath)
 			h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
-			require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
+
+			// Check that the file was transferred.
+			remoteItem := fstest.NewItem("KeyAbsolute", "HELLO", item.ModTime)
+			h.fstestRun.CheckRemoteItems(t, remoteItem)
 
 			// Transfer the same absolute path a second time, but with a different key.
-			h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + fileToTransfer)
+			h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + absPath)
 			h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute2")
-			require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute2"))
+
+			// Check that the same file was transferred to a new name.
+			remoteItem2 := fstest.NewItem("KeyAbsolute2", "HELLO", item.ModTime)
+			h.fstestRun.CheckRemoteItems(t, remoteItem, remoteItem2)
 
 			h.requireWriteLine("CHECKPRESENT KeyAbsolute2")
 			h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyAbsolute2")
@@ -668,30 +829,36 @@ var localBackendTestCases = []testCase{
 	{
 		label: "TransferStoreRelative",
 		testProtocolFunc: func(t *testing.T, h *testState) {
-			h.preconfigureServer()
-
 			// Save the current working directory so we can restore it when this
 			// test ends.
 			cwd, err := os.Getwd()
 			require.NoError(t, err)
 
-			require.NoError(t, os.Chdir(t.TempDir()))
+			tempDir := t.TempDir()
+
+			require.NoError(t, os.Chdir(tempDir))
 			t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
 
+			h.preconfigureServer()
+
 			h.requireReadLineExact("VERSION 1")
 			h.requireWriteLine("INITREMOTE")
 			h.requireReadLineExact("INITREMOTE-SUCCESS")
 
-			// Create temp file for transfer with a relative path.
+			item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
-			fileToTransfer := "file.txt"
+			absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
-			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
-			require.FileExists(t, fileToTransfer)
+			relativePath, err := relativeFilepathWorkaround(tempDir, absPath)
-			require.False(t, filepath.IsAbs(fileToTransfer))
+			require.NoError(t, err)
+			require.False(t, filepath.IsAbs(relativePath))
+			require.FileExists(t, relativePath)
 
 			// Specify a relative path to transfer.
-			h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
+			h.requireWriteLine("TRANSFER STORE KeyRelative " + relativePath)
 			h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
-			require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
+
+			remoteItem := fstest.NewItem("KeyRelative", "HELLO", item.ModTime)
+			h.fstestRun.CheckRemoteItems(t, remoteItem)
 
 			h.requireWriteLine("CHECKPRESENT KeyRelative")
 			h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
@@ -710,7 +877,8 @@ var localBackendTestCases = []testCase{
 			cwd, err := os.Getwd()
 			require.NoError(t, err)
 
-			require.NoError(t, os.Chdir(t.TempDir()))
+			tempDir := t.TempDir()
+			require.NoError(t, os.Chdir(tempDir))
 			t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
 
 			h.preconfigureServer()
@@ -720,15 +888,19 @@ var localBackendTestCases = []testCase{
 			h.requireReadLineExact("INITREMOTE-SUCCESS")
 
 			// Create temp file for transfer.
-			fileToTransfer := "filename with spaces.txt"
+			item := h.fstestRun.WriteFile("filename with spaces.txt", "HELLO", time.Now())
-			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+			absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
-			require.FileExists(t, fileToTransfer)
+			relativePath, err := relativeFilepathWorkaround(tempDir, absPath)
-			require.False(t, filepath.IsAbs(fileToTransfer))
+			require.NoError(t, err)
+			require.False(t, filepath.IsAbs(relativePath))
+			require.FileExists(t, relativePath)
 
 			// Specify a relative path to transfer.
-			h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
+			h.requireWriteLine("TRANSFER STORE KeyRelative " + relativePath)
 			h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
-			require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
+
+			remoteItem := fstest.NewItem("KeyRelative", "HELLO", item.ModTime)
+			h.fstestRun.CheckRemoteItems(t, remoteItem)
 
 			h.requireWriteLine("CHECKPRESENT KeyRelative")
 			h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
@@ -745,8 +917,9 @@ var localBackendTestCases = []testCase{
 			h.preconfigureServer()
 
 			// Create temp file for transfer.
-			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
-			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+			absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+			require.True(t, filepath.IsAbs(absPath))
 
 			h.requireReadLineExact("VERSION 1")
 			h.requireWriteLine("INITREMOTE")
@@ -756,10 +929,11 @@ var localBackendTestCases = []testCase{
 			h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
 
 			// Specify an absolute path to transfer.
-			require.True(t, filepath.IsAbs(fileToTransfer))
+			h.requireWriteLine("TRANSFER STORE KeyAbsolute " + absPath)
-			h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
 			h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
-			require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
+
+			remoteItem := fstest.NewItem("KeyAbsolute", "HELLO", item.ModTime)
+			h.fstestRun.CheckRemoteItems(t, remoteItem)
 
 			require.NoError(t, h.mockStdinW.Close())
 		},
@@ -781,8 +955,9 @@ var localBackendTestCases = []testCase{
 			h.preconfigureServer()
 
 			// Create temp file for transfer.
-			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
-			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+			absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+			require.True(t, filepath.IsAbs(absPath))
 
 			h.requireReadLineExact("VERSION 1")
 			h.requireWriteLine("INITREMOTE")
@@ -791,9 +966,11 @@ var localBackendTestCases = []testCase{
 			h.requireWriteLine("CHECKPRESENT foo")
 			h.requireReadLineExact("CHECKPRESENT-FAILURE foo")
 
-			h.requireWriteLine("TRANSFER STORE foo " + fileToTransfer)
+			h.requireWriteLine("TRANSFER STORE foo " + absPath)
 			h.requireReadLineExact("TRANSFER-SUCCESS STORE foo")
-			require.FileExists(t, filepath.Join(h.localFsDir, "foo"))
+
+			remoteItem := fstest.NewItem("foo", "HELLO", item.ModTime)
+			h.fstestRun.CheckRemoteItems(t, remoteItem)
 
 			h.requireWriteLine("CHECKPRESENT foo")
 			h.requireReadLineExact("CHECKPRESENT-SUCCESS foo")
@@ -807,8 +984,9 @@ var localBackendTestCases = []testCase{
 			h.preconfigureServer()
 
 			// Create temp file for transfer.
-			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
-			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+			absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+			require.True(t, filepath.IsAbs(absPath))
 
 			h.requireReadLineExact("VERSION 1")
 			h.requireWriteLine("INITREMOTE")
@@ -817,10 +995,11 @@ var localBackendTestCases = []testCase{
 			realisticKey := "SHA256E-s1048576--7ba87e06b9b7903cfbaf4a38736766c161e3e7b42f06fe57f040aa410a8f0701.this-is-a-test-key"
 
 			// Specify an absolute path to transfer.
-			require.True(t, filepath.IsAbs(fileToTransfer))
+			h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, absPath))
-			h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, fileToTransfer))
 			h.requireReadLineExact("TRANSFER-SUCCESS STORE " + realisticKey)
-			require.FileExists(t, filepath.Join(h.localFsDir, realisticKey))
+
+			remoteItem := fstest.NewItem(realisticKey, "HELLO", item.ModTime)
+			h.fstestRun.CheckRemoteItems(t, remoteItem)
 
 			h.requireWriteLine("CHECKPRESENT " + realisticKey)
 			h.requireReadLineExact("CHECKPRESENT-SUCCESS " + realisticKey)
@@ -849,27 +1028,36 @@ var localBackendTestCases = []testCase{
 			h.preconfigureServer()
 
 			// Create temp file for transfer.
-			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
-			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+			absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+			require.True(t, filepath.IsAbs(absPath))
 
 			h.requireReadLineExact("VERSION 1")
 			h.requireWriteLine("INITREMOTE")
 			h.requireReadLineExact("INITREMOTE-SUCCESS")
 
 			// Specify an absolute path to transfer.
-			require.True(t, filepath.IsAbs(fileToTransfer))
+			h.requireWriteLine("TRANSFER STORE SomeKey " + absPath)
-			h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
 			h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
-			require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+			remoteItem := fstest.NewItem("SomeKey", "HELLO", item.ModTime)
+			h.fstestRun.CheckRemoteItems(t, remoteItem)
 
 			h.requireWriteLine("CHECKPRESENT SomeKey")
 			h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
 
-			retrievedFilePath := fileToTransfer + ".retrieved"
+			h.fstestRun.CheckLocalItems(t,
-			require.NoFileExists(t, retrievedFilePath)
+				fstest.NewItem("file.txt", "HELLO", item.ModTime),
+			)
+
+			retrievedFilePath := absPath + ".retrieved"
 			h.requireWriteLine("TRANSFER RETRIEVE SomeKey " + retrievedFilePath)
 			h.requireReadLineExact("TRANSFER-SUCCESS RETRIEVE SomeKey")
-			require.FileExists(t, retrievedFilePath)
+
+			h.fstestRun.CheckLocalItems(t,
+				fstest.NewItem("file.txt", "HELLO", item.ModTime),
+				fstest.NewItem("file.txt.retrieved", "HELLO", item.ModTime),
+			)
 
 			require.NoError(t, h.mockStdinW.Close())
 		},
@@ -879,11 +1067,13 @@ var localBackendTestCases = []testCase{
 		testProtocolFunc: func(t *testing.T, h *testState) {
 			h.preconfigureServer()
 
+			ctx := context.WithoutCancel(context.Background())
+
 			// Write a file into the remote without using the git-annex
 			// protocol.
-			remoteFilePath := filepath.Join(h.localFsDir, "SomeKey")
+			remoteItem := h.fstestRun.WriteObject(ctx, "SomeKey", "HELLO", time.Now())
-			require.NoError(t, os.WriteFile(remoteFilePath, []byte("HELLO"), 0600))
+
-			require.FileExists(t, remoteFilePath)
+			h.fstestRun.CheckRemoteItems(t, remoteItem)
 
 			h.requireReadLineExact("VERSION 1")
 			h.requireWriteLine("INITREMOTE")
@@ -891,15 +1081,18 @@ var localBackendTestCases = []testCase{
 
 			h.requireWriteLine("CHECKPRESENT SomeKey")
 			h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
-			require.FileExists(t, remoteFilePath)
+
+			h.fstestRun.CheckRemoteItems(t, remoteItem)
 
 			h.requireWriteLine("REMOVE SomeKey")
 			h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
-			require.NoFileExists(t, remoteFilePath)
+
+			h.requireRemoteIsEmpty()
 
 			h.requireWriteLine("CHECKPRESENT SomeKey")
 			h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
-			require.NoFileExists(t, remoteFilePath)
+
+			h.requireRemoteIsEmpty()
 
 			require.NoError(t, h.mockStdinW.Close())
 		},
@@ -910,8 +1103,9 @@ var localBackendTestCases = []testCase{
 			h.preconfigureServer()
 
 			// Create temp file for transfer.
-			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+			item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
-			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+			absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+			require.True(t, filepath.IsAbs(absPath))
 
 			h.requireReadLineExact("VERSION 1")
 			h.requireWriteLine("INITREMOTE")
@@ -921,17 +1115,19 @@ var localBackendTestCases = []testCase{
 			h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
 
 			// Specify an absolute path to transfer.
-			require.True(t, filepath.IsAbs(fileToTransfer))
+			h.requireWriteLine("TRANSFER STORE SomeKey " + absPath)
-			h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
 			h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
-			require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+			remoteItem := fstest.NewItem("SomeKey", "HELLO", item.ModTime)
+			h.fstestRun.CheckRemoteItems(t, remoteItem)
 
 			h.requireWriteLine("CHECKPRESENT SomeKey")
 			h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
 
 			h.requireWriteLine("REMOVE SomeKey")
 			h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
-			require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+			h.requireRemoteIsEmpty()
 
 			h.requireWriteLine("CHECKPRESENT SomeKey")
 			h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
@@ -944,10 +1140,6 @@ var localBackendTestCases = []testCase{
 		testProtocolFunc: func(t *testing.T, h *testState) {
 			h.preconfigureServer()
 
-			// Create temp file for transfer.
-			fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
-			require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
-
 			h.requireReadLineExact("VERSION 1")
 			h.requireWriteLine("INITREMOTE")
 			h.requireReadLineExact("INITREMOTE-SUCCESS")
@@ -955,10 +1147,12 @@ var localBackendTestCases = []testCase{
 			h.requireWriteLine("CHECKPRESENT SomeKey")
 			h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
 
-			require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+			h.requireRemoteIsEmpty()
+
 			h.requireWriteLine("REMOVE SomeKey")
 			h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
-			require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+			h.requireRemoteIsEmpty()
 
 			h.requireWriteLine("CHECKPRESENT SomeKey")
 			h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
@@ -983,44 +1177,40 @@ var localBackendTestCases = []testCase{
 	},
 }
 
-func TestGitAnnexLocalBackendCases(t *testing.T) {
+// TestMain drives the tests
-	for _, testCase := range localBackendTestCases {
+func TestMain(m *testing.M) {
-		// Clear global state left behind by tests that chdir to a temp directory.
+	fstest.TestMain(m)
-		cache.Clear()
+}
 
+// Run fstest-compatible test cases with backend selected by `-remote`.
+func TestGitAnnexFstestBackendCases(t *testing.T) {
+
+	for _, testCase := range fstestTestCases {
 		// TODO: Remove this when rclone requires a Go version >= 1.22. Future
 		// versions of Go fix the semantics of capturing a range variable.
 		// https://go.dev/blog/loopvar-preview
 		testCase := testCase
 
 		t.Run(testCase.label, func(t *testing.T) {
-			tempDir := t.TempDir()
+			r := fstest.NewRun(t)
+			t.Cleanup(func() { r.Finalise() })
 
-			// Create temp dir for an rclone remote pointing at local filesystem.
+			// Parse the fstest-provided remote string. It might have a path!
-			localFsDir := filepath.Join(tempDir, "remoteTarget")
+			remoteName, remotePath, err := fspath.SplitFs(r.FremoteName)
-			require.NoError(t, os.Mkdir(localFsDir, 0700))
+			require.NoError(t, err)
 
-			// Create temp config
+			// The gitannex command requires the `rcloneremotename` is the name
-			remoteName := getUniqueRemoteName(t)
+			// of a remote or exactly ":local", so the empty string will not
-			configLines := []string{
+			// suffice.
-				fmt.Sprintf("[%s]", remoteName),
+			if remoteName == "" {
-				"type = local",
+				require.True(t, r.Fremote.Features().IsLocal)
-				fmt.Sprintf("remote = %s", localFsDir),
+				remoteName = ":local"
 			}
-			configContents := strings.Join(configLines, "\n")
-
-			configPath := filepath.Join(tempDir, "rclone.conf")
-			require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
-			require.NoError(t, config.SetConfigPath(configPath))
-
-			// The custom config file will be ignored unless we install the
-			// global config file handler.
-			configfile.Install()
 
 			handle := makeTestState(t)
-			handle.localFsDir = localFsDir
+			handle.fstestRun = r
-			handle.configPath = configPath
 			handle.remoteName = remoteName
+			handle.remotePrefix = remotePath
 
 			var wg sync.WaitGroup
 			wg.Add(1)
@@ -1042,54 +1232,3 @@ func TestGitAnnexLocalBackendCases(t *testing.T) {
 		})
 	}
 }
-
-// Configure the git-annex client with a mockfs backend and send it the
-// "INITREMOTE" command over mocked stdin. This should fail because mockfs does
-// not support empty directories.
-func TestGitAnnexHandleInitRemoteBackendDoesNotSupportEmptyDirectories(t *testing.T) {
-	tempDir := t.TempDir()
-
-	// Temporarily override the filesystem registry.
-	oldRegistry := fs.Registry
-	mockfs.Register()
-	defer func() { fs.Registry = oldRegistry }()
-
-	// Create temp dir for an rclone remote pointing at local filesystem.
-	localFsDir := filepath.Join(tempDir, "remoteTarget")
-	require.NoError(t, os.Mkdir(localFsDir, 0700))
-
-	// Create temp config
-	remoteName := getUniqueRemoteName(t)
-	configLines := []string{
-		fmt.Sprintf("[%s]", remoteName),
-		"type = mockfs",
-		fmt.Sprintf("remote = %s", localFsDir),
-	}
-	configContents := strings.Join(configLines, "\n")
-
-	configPath := filepath.Join(tempDir, "rclone.conf")
-	require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
-
-	// The custom config file will be ignored unless we install the global
-	// config file handler.
-	configfile.Install()
-	require.NoError(t, config.SetConfigPath(configPath))
-
-	handle := makeTestState(t)
-	handle.server.configPrefix = localFsDir
-	handle.server.configRcloneRemoteName = remoteName
-	handle.server.configsDone = true
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-
-	go func() {
-		require.NotNil(t, handle.server.run())
-		wg.Done()
-	}()
-	defer wg.Wait()
-
-	handle.requireReadLineExact("VERSION 1")
-	handle.requireWriteLine("INITREMOTE")
-	handle.requireReadLineExact("INITREMOTE-FAILURE this rclone remote does not support empty directories")
-}

cmd/gitannex/layout.go

@@ -3,6 +3,8 @@ package gitannex
 import (
 	"fmt"
 	"strings"
+
+	"github.com/rclone/rclone/fs/fspath"
 )
 
 type layoutMode string
@@ -39,8 +41,11 @@ func parseLayoutMode(mode string) layoutMode {
 type queryDirhashFunc func(msg string) (string, error)
 
 func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteName, prefix string) (string, error) {
+	remoteName = strings.TrimSuffix(remoteName, ":") + ":"
+	remoteString := fspath.JoinRootPath(remoteName, prefix)
+
 	if mode == layoutModeNodir {
-		return fmt.Sprintf("%s:%s", remoteName, prefix), nil
+		return remoteString, nil
 	}
 
 	var dirhash string
@@ -59,13 +64,13 @@ func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteNa
 
 	switch mode {
 	case layoutModeLower:
-		return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
+		return fmt.Sprintf("%s/%s", remoteString, dirhash), nil
 	case layoutModeDirectory:
-		return fmt.Sprintf("%s:%s/%s%s", remoteName, prefix, dirhash, key), nil
+		return fmt.Sprintf("%s/%s%s", remoteString, dirhash, key), nil
 	case layoutModeMixed:
-		return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
+		return fmt.Sprintf("%s/%s", remoteString, dirhash), nil
 	case layoutModeFrankencase:
-		return fmt.Sprintf("%s:%s/%s", remoteName, prefix, strings.ToLower(dirhash)), nil
+		return fmt.Sprintf("%s/%s", remoteString, strings.ToLower(dirhash)), nil
 	default:
 		panic("unreachable")
 	}