diff --git a/cmd/gitannex/gitannex.go b/cmd/gitannex/gitannex.go index 287adc7ae..d71393b05 100644 --- a/cmd/gitannex/gitannex.go +++ b/cmd/gitannex/gitannex.go @@ -7,7 +7,7 @@ // (Tracked in [issue #7625].) // // 1. ✅ Minimal support for the [external special remote protocol]. Tested on -// "local" and "drive" backends. +// "local", "drive", and "dropbox" backends. // 2. Add support for the ASYNC protocol extension. This may improve performance. // 3. Support the [simple export interface]. This will enable `git-annex // export` functionality. @@ -33,6 +33,7 @@ import ( "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/operations" "github.com/spf13/cobra" ) @@ -267,21 +268,33 @@ func (s *server) handleInitRemote() error { return fmt.Errorf("failed to get configs: %w", err) } - remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName)) - if err != nil { - s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote") - return fmt.Errorf("failed to open root directory of rclone remote: %w", err) + // Explicitly check that a remote with the given name exists. If we just + // relied on `cache.Get()` to return `fs.ErrorNotFoundInConfigFile`, this + // function would incorrectly succeed when the given remote name is actually + // a file path. + // + // The :local: backend does not correspond to a remote named by the rclone + // config, but is permitted to enable testing. Technically, a user might hit + // this code path, but it would be a strange choice because git-annex + // natively supports a "directory" special remote. + + trimmedName := strings.TrimSuffix(s.configRcloneRemoteName, ":") + + if s.configRcloneRemoteName != ":local" { + var remoteExists bool + for _, remoteName := range config.FileSections() { + if remoteName == trimmedName { + remoteExists = true + break + } + } + if !remoteExists { + s.sendMsg("INITREMOTE-FAILURE remote does not exist: " + s.configRcloneRemoteName) + return fmt.Errorf("remote does not exist: %s", s.configRcloneRemoteName) + } } - if !remoteRootFs.Features().CanHaveEmptyDirectories { - s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories") - return fmt.Errorf("rclone remote does not support empty directories") - } - - if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil { - s.sendMsg("INITREMOTE-FAILURE failed to mkdir") - return fmt.Errorf("failed to mkdir: %w", err) - } + s.configRcloneRemoteName = trimmedName + ":" s.sendMsg("INITREMOTE-SUCCESS") return nil diff --git a/cmd/gitannex/gitannex_test.go b/cmd/gitannex/gitannex_test.go index d53a7f921..6d60e1db1 100644 --- a/cmd/gitannex/gitannex_test.go +++ b/cmd/gitannex/gitannex_test.go @@ -2,25 +2,24 @@ package gitannex import ( "bufio" - "crypto/sha256" + "context" "fmt" "io" "os" "path/filepath" "regexp" + "runtime" "strings" "sync" "testing" + "time" - // Without this import, the local filesystem backend would be unavailable. - // It looks unused, but the act of importing it runs its `init()` function. - _ "github.com/rclone/rclone/backend/local" + // Without this import, the various backends would be unavailable. It looks + // unused, but the act of importing runs the package's `init()` function. + _ "github.com/rclone/rclone/backend/all" - "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/cache" - "github.com/rclone/rclone/fs/config" - "github.com/rclone/rclone/fs/config/configfile" - "github.com/rclone/rclone/fstest/mockfs" + "github.com/rclone/rclone/fs/fspath" + "github.com/rclone/rclone/fstest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -256,9 +255,9 @@ type testState struct { mockStdinW *io.PipeWriter mockStdoutReader *bufio.Reader - localFsDir string - configPath string - remoteName string + fstestRun *fstest.Run + remoteName string + remotePrefix string } func makeTestState(t *testing.T) testState { @@ -276,6 +275,10 @@ func makeTestState(t *testing.T) testState { } } +func (h *testState) requireRemoteIsEmpty() { + h.fstestRun.CheckRemoteItems(h.t) +} + func (h *testState) requireReadLineExact(line string) { receivedLine, err := h.mockStdoutReader.ReadString('\n') require.NoError(h.t, err) @@ -296,21 +299,106 @@ func (h *testState) requireWriteLine(line string) { // Preconfigure the handle. This enables the calling test to skip the PREPARE // handshake. func (h *testState) preconfigureServer() { - h.server.configPrefix = h.localFsDir h.server.configRcloneRemoteName = h.remoteName + h.server.configPrefix = h.remotePrefix h.server.configRcloneLayout = string(layoutModeNodir) h.server.configsDone = true } -// getUniqueRemoteName returns a valid remote name derived from the given test's -// name. This is necessary because when a test registers a second remote with -// the same name, the original remote appears to take precedence. This function -// is injective, so each test gets a unique remote name. Returned strings -// contain no spaces. -func getUniqueRemoteName(t *testing.T) string { - // Using sha256 as a hack to ensure injectivity without adding a global - // variable. - return fmt.Sprintf("remote-%x", sha256.Sum256([]byte(t.Name()))) +// Drop-in replacement for `filepath.Rel()` that works around a Windows-specific +// quirk when one of the paths begins with `\\?\` or `//?/`. It seems that +// fstest gives us paths with this prefix on Windows, which throws a wrench in +// the gitannex tests that need to construct relative paths from absolute paths. +// For a demonstration, see `TestWindowsFilepathRelQuirk` below. +// +// The `\\?\` prefix tells Windows APIs to pass strings unmodified to the +// filesystem without additional parsing [1]. Our workaround is roughly to add +// the prefix to whichever parameter doesn't have it (when the OS is Windows). +// I'm not sure this generalizes, but it works for the the kinds of inputs we're +// throwing at it. +// +// [1]: https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file?redirectedfrom=MSDN#win32-file-namespaces +func relativeFilepathWorkaround(basepath, targpath string) (string, error) { + if runtime.GOOS != "windows" { + return filepath.Rel(basepath, targpath) + } + // Canonicalize paths to use backslashes. + basepath = filepath.Clean(basepath) + targpath = filepath.Clean(targpath) + + const winFilePrefixDisableStringParsing = `\\?\` + baseHasPrefix := strings.HasPrefix(basepath, winFilePrefixDisableStringParsing) + targHasPrefix := strings.HasPrefix(targpath, winFilePrefixDisableStringParsing) + + if baseHasPrefix && !targHasPrefix { + targpath = winFilePrefixDisableStringParsing + targpath + } + if !baseHasPrefix && targHasPrefix { + basepath = winFilePrefixDisableStringParsing + basepath + } + return filepath.Rel(basepath, targpath) +} + +func TestWindowsFilepathRelQuirk(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip() + } + + t.Run("filepathRelQuirk", func(t *testing.T) { + var err error + + _, err = filepath.Rel(`C:\foo`, `\\?\C:\foo\bar`) + require.Error(t, err) + + _, err = filepath.Rel(`C:/foo`, `//?/C:/foo/bar`) + require.Error(t, err) + + _, err = filepath.Rel(`\\?\C:\foo`, `C:\foo\bar`) + require.Error(t, err) + + _, err = filepath.Rel(`//?/C:/foo`, `C:/foo/bar`) + require.Error(t, err) + + path, err := filepath.Rel(`\\?\C:\foo`, `\\?\C:\foo\bar`) + require.NoError(t, err) + require.Equal(t, path, `bar`) + + path, err = filepath.Rel(`//?/C:/foo`, `//?/C:/foo/bar`) + require.NoError(t, err) + require.Equal(t, path, `bar`) + }) + + t.Run("fstestAndTempDirHaveDifferentPrefixes", func(t *testing.T) { + r := fstest.NewRun(t) + p := r.Flocal.Root() + require.True(t, strings.HasPrefix(p, `//?/`)) + + tempDir := t.TempDir() + require.False(t, strings.HasPrefix(tempDir, `//?/`)) + require.False(t, strings.HasPrefix(tempDir, `\\?\`)) + }) + + t.Run("workaroundWorks", func(t *testing.T) { + path, err := relativeFilepathWorkaround(`C:\foo`, `\\?\C:\foo\bar`) + require.NoError(t, err) + require.Equal(t, path, "bar") + + path, err = relativeFilepathWorkaround(`C:/foo`, `//?/C:/foo/bar`) + require.NoError(t, err) + require.Equal(t, path, "bar") + + path, err = relativeFilepathWorkaround(`\\?\C:\foo`, `C:\foo\bar`) + require.NoError(t, err) + require.Equal(t, path, `bar`) + + path, err = relativeFilepathWorkaround(`//?/C:/foo`, `C:/foo/bar`) + require.NoError(t, err) + require.Equal(t, path, `bar`) + + path, err = relativeFilepathWorkaround(`\\?\C:\foo`, `\\?\C:\foo\bar`) + require.NoError(t, err) + require.Equal(t, path, `bar`) + }) } type testCase struct { @@ -319,8 +407,8 @@ type testCase struct { expectedError string } -// These test cases run against the "local" backend. -var localBackendTestCases = []testCase{ +// These test cases run against a backend selected by the `-remote` flag. +var fstestTestCases = []testCase{ { label: "HandlesInit", testProtocolFunc: func(t *testing.T, h *testState) { @@ -377,18 +465,86 @@ var localBackendTestCases = []testCase{ h.requireReadLineExact("GETCONFIG rcloneremotename") h.requireWriteLine("VALUE " + h.remoteName) h.requireReadLineExact("GETCONFIG rcloneprefix") - h.requireWriteLine("VALUE " + h.localFsDir) + h.requireWriteLine("VALUE " + h.remotePrefix) h.requireReadLineExact("GETCONFIG rclonelayout") h.requireWriteLine("VALUE foo") h.requireReadLineExact("PREPARE-SUCCESS") require.Equal(t, h.server.configRcloneRemoteName, h.remoteName) - require.Equal(t, h.server.configPrefix, h.localFsDir) + require.Equal(t, h.server.configPrefix, h.remotePrefix) require.True(t, h.server.configsDone) require.NoError(t, h.mockStdinW.Close()) }, }, + { + label: "HandlesPrepareWithNonexistentRemote", + testProtocolFunc: func(t *testing.T, h *testState) { + h.requireReadLineExact("VERSION 1") + h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension + h.requireReadLineExact("EXTENSIONS") + + if !h.server.extensionInfo { + t.Errorf("expected INFO extension to be enabled") + return + } + + h.requireWriteLine("PREPARE") + h.requireReadLineExact("GETCONFIG rcloneremotename") + h.requireWriteLine("VALUE thisRemoteDoesNotExist") + h.requireReadLineExact("GETCONFIG rcloneprefix") + h.requireWriteLine("VALUE " + h.remotePrefix) + h.requireReadLineExact("GETCONFIG rclonelayout") + h.requireWriteLine("VALUE foo") + h.requireReadLineExact("PREPARE-SUCCESS") + + require.Equal(t, h.server.configRcloneRemoteName, "thisRemoteDoesNotExist") + require.Equal(t, h.server.configPrefix, h.remotePrefix) + require.True(t, h.server.configsDone) + + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-FAILURE remote does not exist: thisRemoteDoesNotExist") + + require.NoError(t, h.mockStdinW.Close()) + }, + expectedError: "remote does not exist: thisRemoteDoesNotExist", + }, + { + label: "HandlesPrepareWithPathAsRemote", + testProtocolFunc: func(t *testing.T, h *testState) { + h.requireReadLineExact("VERSION 1") + h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension + h.requireReadLineExact("EXTENSIONS") + + if !h.server.extensionInfo { + t.Errorf("expected INFO extension to be enabled") + return + } + + h.requireWriteLine("PREPARE") + h.requireReadLineExact("GETCONFIG rcloneremotename") + h.requireWriteLine("VALUE " + h.remotePrefix) + h.requireReadLineExact("GETCONFIG rcloneprefix") + h.requireWriteLine("VALUE /foo") + h.requireReadLineExact("GETCONFIG rclonelayout") + h.requireWriteLine("VALUE foo") + h.requireReadLineExact("PREPARE-SUCCESS") + + require.Equal(t, h.server.configRcloneRemoteName, h.remotePrefix) + require.Equal(t, h.server.configPrefix, "/foo") + require.True(t, h.server.configsDone) + + h.requireWriteLine("INITREMOTE") + + require.Regexp(t, + regexp.MustCompile("^INITREMOTE-FAILURE remote does not exist: "), + h.requireReadLine(), + ) + + require.NoError(t, h.mockStdinW.Close()) + }, + expectedError: "remote does not exist:", + }, { label: "HandlesPrepareWithSynonyms", testProtocolFunc: func(t *testing.T, h *testState) { @@ -409,13 +565,13 @@ var localBackendTestCases = []testCase{ h.requireWriteLine("VALUE " + h.remoteName) h.requireReadLineExact("GETCONFIG rcloneprefix") - h.requireWriteLine("VALUE " + h.localFsDir) + h.requireWriteLine("VALUE " + h.remotePrefix) h.requireReadLineExact("GETCONFIG rclonelayout") h.requireWriteLine("VALUE foo") h.requireReadLineExact("PREPARE-SUCCESS") require.Equal(t, h.server.configRcloneRemoteName, h.remoteName) - require.Equal(t, h.server.configPrefix, h.localFsDir) + require.Equal(t, h.server.configPrefix, h.remotePrefix) require.True(t, h.server.configsDone) require.NoError(t, h.mockStdinW.Close()) @@ -437,12 +593,12 @@ var localBackendTestCases = []testCase{ h.requireReadLineExact("GETCONFIG rcloneremotename") remoteNameWithSpaces := fmt.Sprintf(" %s ", h.remoteName) - localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir) + prefixWithWhitespace := fmt.Sprintf(" %s\t", h.remotePrefix) h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces)) h.requireReadLineExact("GETCONFIG rcloneprefix") - h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces)) + h.requireWriteLine(fmt.Sprintf("VALUE %s", prefixWithWhitespace)) h.requireReadLineExact("GETCONFIG rclonelayout") h.requireWriteLine("VALUE") @@ -452,7 +608,7 @@ var localBackendTestCases = []testCase{ h.requireReadLineExact("PREPARE-SUCCESS") require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces) - require.Equal(t, h.server.configPrefix, localFsDirWithSpaces) + require.Equal(t, h.server.configPrefix, prefixWithWhitespace) require.True(t, h.server.configsDone) require.NoError(t, h.mockStdinW.Close()) @@ -639,20 +795,25 @@ var localBackendTestCases = []testCase{ h.requireReadLineExact("INITREMOTE-SUCCESS") // Create temp file for transfer with an absolute path. - fileToTransfer := filepath.Join(t.TempDir(), "file.txt") - require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) - require.FileExists(t, fileToTransfer) - require.True(t, filepath.IsAbs(fileToTransfer)) + item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now()) + absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path) + require.True(t, filepath.IsAbs(absPath)) // Specify an absolute path to transfer. - h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer) + h.requireWriteLine("TRANSFER STORE KeyAbsolute " + absPath) h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute") - require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute")) + + // Check that the file was transferred. + remoteItem := fstest.NewItem("KeyAbsolute", "HELLO", item.ModTime) + h.fstestRun.CheckRemoteItems(t, remoteItem) // Transfer the same absolute path a second time, but with a different key. - h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + fileToTransfer) + h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + absPath) h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute2") - require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute2")) + + // Check that the same file was transferred to a new name. + remoteItem2 := fstest.NewItem("KeyAbsolute2", "HELLO", item.ModTime) + h.fstestRun.CheckRemoteItems(t, remoteItem, remoteItem2) h.requireWriteLine("CHECKPRESENT KeyAbsolute2") h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyAbsolute2") @@ -668,30 +829,36 @@ var localBackendTestCases = []testCase{ { label: "TransferStoreRelative", testProtocolFunc: func(t *testing.T, h *testState) { - h.preconfigureServer() - // Save the current working directory so we can restore it when this // test ends. cwd, err := os.Getwd() require.NoError(t, err) - require.NoError(t, os.Chdir(t.TempDir())) + tempDir := t.TempDir() + + require.NoError(t, os.Chdir(tempDir)) t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) }) + h.preconfigureServer() + h.requireReadLineExact("VERSION 1") h.requireWriteLine("INITREMOTE") h.requireReadLineExact("INITREMOTE-SUCCESS") - // Create temp file for transfer with a relative path. - fileToTransfer := "file.txt" - require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) - require.FileExists(t, fileToTransfer) - require.False(t, filepath.IsAbs(fileToTransfer)) + item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now()) + absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path) + + relativePath, err := relativeFilepathWorkaround(tempDir, absPath) + require.NoError(t, err) + require.False(t, filepath.IsAbs(relativePath)) + require.FileExists(t, relativePath) // Specify a relative path to transfer. - h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer) + h.requireWriteLine("TRANSFER STORE KeyRelative " + relativePath) h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative") - require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative")) + + remoteItem := fstest.NewItem("KeyRelative", "HELLO", item.ModTime) + h.fstestRun.CheckRemoteItems(t, remoteItem) h.requireWriteLine("CHECKPRESENT KeyRelative") h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative") @@ -710,7 +877,8 @@ var localBackendTestCases = []testCase{ cwd, err := os.Getwd() require.NoError(t, err) - require.NoError(t, os.Chdir(t.TempDir())) + tempDir := t.TempDir() + require.NoError(t, os.Chdir(tempDir)) t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) }) h.preconfigureServer() @@ -720,15 +888,19 @@ var localBackendTestCases = []testCase{ h.requireReadLineExact("INITREMOTE-SUCCESS") // Create temp file for transfer. - fileToTransfer := "filename with spaces.txt" - require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) - require.FileExists(t, fileToTransfer) - require.False(t, filepath.IsAbs(fileToTransfer)) + item := h.fstestRun.WriteFile("filename with spaces.txt", "HELLO", time.Now()) + absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path) + relativePath, err := relativeFilepathWorkaround(tempDir, absPath) + require.NoError(t, err) + require.False(t, filepath.IsAbs(relativePath)) + require.FileExists(t, relativePath) // Specify a relative path to transfer. - h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer) + h.requireWriteLine("TRANSFER STORE KeyRelative " + relativePath) h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative") - require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative")) + + remoteItem := fstest.NewItem("KeyRelative", "HELLO", item.ModTime) + h.fstestRun.CheckRemoteItems(t, remoteItem) h.requireWriteLine("CHECKPRESENT KeyRelative") h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative") @@ -745,8 +917,9 @@ var localBackendTestCases = []testCase{ h.preconfigureServer() // Create temp file for transfer. - fileToTransfer := filepath.Join(t.TempDir(), "file.txt") - require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now()) + absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path) + require.True(t, filepath.IsAbs(absPath)) h.requireReadLineExact("VERSION 1") h.requireWriteLine("INITREMOTE") @@ -756,10 +929,11 @@ var localBackendTestCases = []testCase{ h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist") // Specify an absolute path to transfer. - require.True(t, filepath.IsAbs(fileToTransfer)) - h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer) + h.requireWriteLine("TRANSFER STORE KeyAbsolute " + absPath) h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute") - require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute")) + + remoteItem := fstest.NewItem("KeyAbsolute", "HELLO", item.ModTime) + h.fstestRun.CheckRemoteItems(t, remoteItem) require.NoError(t, h.mockStdinW.Close()) }, @@ -781,8 +955,9 @@ var localBackendTestCases = []testCase{ h.preconfigureServer() // Create temp file for transfer. - fileToTransfer := filepath.Join(t.TempDir(), "file.txt") - require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now()) + absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path) + require.True(t, filepath.IsAbs(absPath)) h.requireReadLineExact("VERSION 1") h.requireWriteLine("INITREMOTE") @@ -791,9 +966,11 @@ var localBackendTestCases = []testCase{ h.requireWriteLine("CHECKPRESENT foo") h.requireReadLineExact("CHECKPRESENT-FAILURE foo") - h.requireWriteLine("TRANSFER STORE foo " + fileToTransfer) + h.requireWriteLine("TRANSFER STORE foo " + absPath) h.requireReadLineExact("TRANSFER-SUCCESS STORE foo") - require.FileExists(t, filepath.Join(h.localFsDir, "foo")) + + remoteItem := fstest.NewItem("foo", "HELLO", item.ModTime) + h.fstestRun.CheckRemoteItems(t, remoteItem) h.requireWriteLine("CHECKPRESENT foo") h.requireReadLineExact("CHECKPRESENT-SUCCESS foo") @@ -807,8 +984,9 @@ var localBackendTestCases = []testCase{ h.preconfigureServer() // Create temp file for transfer. - fileToTransfer := filepath.Join(t.TempDir(), "file.txt") - require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now()) + absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path) + require.True(t, filepath.IsAbs(absPath)) h.requireReadLineExact("VERSION 1") h.requireWriteLine("INITREMOTE") @@ -817,10 +995,11 @@ var localBackendTestCases = []testCase{ realisticKey := "SHA256E-s1048576--7ba87e06b9b7903cfbaf4a38736766c161e3e7b42f06fe57f040aa410a8f0701.this-is-a-test-key" // Specify an absolute path to transfer. - require.True(t, filepath.IsAbs(fileToTransfer)) - h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, fileToTransfer)) + h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, absPath)) h.requireReadLineExact("TRANSFER-SUCCESS STORE " + realisticKey) - require.FileExists(t, filepath.Join(h.localFsDir, realisticKey)) + + remoteItem := fstest.NewItem(realisticKey, "HELLO", item.ModTime) + h.fstestRun.CheckRemoteItems(t, remoteItem) h.requireWriteLine("CHECKPRESENT " + realisticKey) h.requireReadLineExact("CHECKPRESENT-SUCCESS " + realisticKey) @@ -849,27 +1028,36 @@ var localBackendTestCases = []testCase{ h.preconfigureServer() // Create temp file for transfer. - fileToTransfer := filepath.Join(t.TempDir(), "file.txt") - require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now()) + absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path) + require.True(t, filepath.IsAbs(absPath)) h.requireReadLineExact("VERSION 1") h.requireWriteLine("INITREMOTE") h.requireReadLineExact("INITREMOTE-SUCCESS") // Specify an absolute path to transfer. - require.True(t, filepath.IsAbs(fileToTransfer)) - h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer) + h.requireWriteLine("TRANSFER STORE SomeKey " + absPath) h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey") - require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey")) + + remoteItem := fstest.NewItem("SomeKey", "HELLO", item.ModTime) + h.fstestRun.CheckRemoteItems(t, remoteItem) h.requireWriteLine("CHECKPRESENT SomeKey") h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey") - retrievedFilePath := fileToTransfer + ".retrieved" - require.NoFileExists(t, retrievedFilePath) + h.fstestRun.CheckLocalItems(t, + fstest.NewItem("file.txt", "HELLO", item.ModTime), + ) + + retrievedFilePath := absPath + ".retrieved" h.requireWriteLine("TRANSFER RETRIEVE SomeKey " + retrievedFilePath) h.requireReadLineExact("TRANSFER-SUCCESS RETRIEVE SomeKey") - require.FileExists(t, retrievedFilePath) + + h.fstestRun.CheckLocalItems(t, + fstest.NewItem("file.txt", "HELLO", item.ModTime), + fstest.NewItem("file.txt.retrieved", "HELLO", item.ModTime), + ) require.NoError(t, h.mockStdinW.Close()) }, @@ -879,11 +1067,13 @@ var localBackendTestCases = []testCase{ testProtocolFunc: func(t *testing.T, h *testState) { h.preconfigureServer() + ctx := context.WithoutCancel(context.Background()) + // Write a file into the remote without using the git-annex // protocol. - remoteFilePath := filepath.Join(h.localFsDir, "SomeKey") - require.NoError(t, os.WriteFile(remoteFilePath, []byte("HELLO"), 0600)) - require.FileExists(t, remoteFilePath) + remoteItem := h.fstestRun.WriteObject(ctx, "SomeKey", "HELLO", time.Now()) + + h.fstestRun.CheckRemoteItems(t, remoteItem) h.requireReadLineExact("VERSION 1") h.requireWriteLine("INITREMOTE") @@ -891,15 +1081,18 @@ var localBackendTestCases = []testCase{ h.requireWriteLine("CHECKPRESENT SomeKey") h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey") - require.FileExists(t, remoteFilePath) + + h.fstestRun.CheckRemoteItems(t, remoteItem) h.requireWriteLine("REMOVE SomeKey") h.requireReadLineExact("REMOVE-SUCCESS SomeKey") - require.NoFileExists(t, remoteFilePath) + + h.requireRemoteIsEmpty() h.requireWriteLine("CHECKPRESENT SomeKey") h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey") - require.NoFileExists(t, remoteFilePath) + + h.requireRemoteIsEmpty() require.NoError(t, h.mockStdinW.Close()) }, @@ -910,8 +1103,9 @@ var localBackendTestCases = []testCase{ h.preconfigureServer() // Create temp file for transfer. - fileToTransfer := filepath.Join(t.TempDir(), "file.txt") - require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now()) + absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path) + require.True(t, filepath.IsAbs(absPath)) h.requireReadLineExact("VERSION 1") h.requireWriteLine("INITREMOTE") @@ -921,17 +1115,19 @@ var localBackendTestCases = []testCase{ h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey") // Specify an absolute path to transfer. - require.True(t, filepath.IsAbs(fileToTransfer)) - h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer) + h.requireWriteLine("TRANSFER STORE SomeKey " + absPath) h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey") - require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey")) + + remoteItem := fstest.NewItem("SomeKey", "HELLO", item.ModTime) + h.fstestRun.CheckRemoteItems(t, remoteItem) h.requireWriteLine("CHECKPRESENT SomeKey") h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey") h.requireWriteLine("REMOVE SomeKey") h.requireReadLineExact("REMOVE-SUCCESS SomeKey") - require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey")) + + h.requireRemoteIsEmpty() h.requireWriteLine("CHECKPRESENT SomeKey") h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey") @@ -944,10 +1140,6 @@ var localBackendTestCases = []testCase{ testProtocolFunc: func(t *testing.T, h *testState) { h.preconfigureServer() - // Create temp file for transfer. - fileToTransfer := filepath.Join(t.TempDir(), "file.txt") - require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) - h.requireReadLineExact("VERSION 1") h.requireWriteLine("INITREMOTE") h.requireReadLineExact("INITREMOTE-SUCCESS") @@ -955,10 +1147,12 @@ var localBackendTestCases = []testCase{ h.requireWriteLine("CHECKPRESENT SomeKey") h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey") - require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey")) + h.requireRemoteIsEmpty() + h.requireWriteLine("REMOVE SomeKey") h.requireReadLineExact("REMOVE-SUCCESS SomeKey") - require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey")) + + h.requireRemoteIsEmpty() h.requireWriteLine("CHECKPRESENT SomeKey") h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey") @@ -983,44 +1177,40 @@ var localBackendTestCases = []testCase{ }, } -func TestGitAnnexLocalBackendCases(t *testing.T) { - for _, testCase := range localBackendTestCases { - // Clear global state left behind by tests that chdir to a temp directory. - cache.Clear() +// TestMain drives the tests +func TestMain(m *testing.M) { + fstest.TestMain(m) +} +// Run fstest-compatible test cases with backend selected by `-remote`. +func TestGitAnnexFstestBackendCases(t *testing.T) { + + for _, testCase := range fstestTestCases { // TODO: Remove this when rclone requires a Go version >= 1.22. Future // versions of Go fix the semantics of capturing a range variable. // https://go.dev/blog/loopvar-preview testCase := testCase t.Run(testCase.label, func(t *testing.T) { - tempDir := t.TempDir() + r := fstest.NewRun(t) + t.Cleanup(func() { r.Finalise() }) - // Create temp dir for an rclone remote pointing at local filesystem. - localFsDir := filepath.Join(tempDir, "remoteTarget") - require.NoError(t, os.Mkdir(localFsDir, 0700)) + // Parse the fstest-provided remote string. It might have a path! + remoteName, remotePath, err := fspath.SplitFs(r.FremoteName) + require.NoError(t, err) - // Create temp config - remoteName := getUniqueRemoteName(t) - configLines := []string{ - fmt.Sprintf("[%s]", remoteName), - "type = local", - fmt.Sprintf("remote = %s", localFsDir), + // The gitannex command requires the `rcloneremotename` is the name + // of a remote or exactly ":local", so the empty string will not + // suffice. + if remoteName == "" { + require.True(t, r.Fremote.Features().IsLocal) + remoteName = ":local" } - configContents := strings.Join(configLines, "\n") - - configPath := filepath.Join(tempDir, "rclone.conf") - require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600)) - require.NoError(t, config.SetConfigPath(configPath)) - - // The custom config file will be ignored unless we install the - // global config file handler. - configfile.Install() handle := makeTestState(t) - handle.localFsDir = localFsDir - handle.configPath = configPath + handle.fstestRun = r handle.remoteName = remoteName + handle.remotePrefix = remotePath var wg sync.WaitGroup wg.Add(1) @@ -1042,54 +1232,3 @@ func TestGitAnnexLocalBackendCases(t *testing.T) { }) } } - -// Configure the git-annex client with a mockfs backend and send it the -// "INITREMOTE" command over mocked stdin. This should fail because mockfs does -// not support empty directories. -func TestGitAnnexHandleInitRemoteBackendDoesNotSupportEmptyDirectories(t *testing.T) { - tempDir := t.TempDir() - - // Temporarily override the filesystem registry. - oldRegistry := fs.Registry - mockfs.Register() - defer func() { fs.Registry = oldRegistry }() - - // Create temp dir for an rclone remote pointing at local filesystem. - localFsDir := filepath.Join(tempDir, "remoteTarget") - require.NoError(t, os.Mkdir(localFsDir, 0700)) - - // Create temp config - remoteName := getUniqueRemoteName(t) - configLines := []string{ - fmt.Sprintf("[%s]", remoteName), - "type = mockfs", - fmt.Sprintf("remote = %s", localFsDir), - } - configContents := strings.Join(configLines, "\n") - - configPath := filepath.Join(tempDir, "rclone.conf") - require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600)) - - // The custom config file will be ignored unless we install the global - // config file handler. - configfile.Install() - require.NoError(t, config.SetConfigPath(configPath)) - - handle := makeTestState(t) - handle.server.configPrefix = localFsDir - handle.server.configRcloneRemoteName = remoteName - handle.server.configsDone = true - - var wg sync.WaitGroup - wg.Add(1) - - go func() { - require.NotNil(t, handle.server.run()) - wg.Done() - }() - defer wg.Wait() - - handle.requireReadLineExact("VERSION 1") - handle.requireWriteLine("INITREMOTE") - handle.requireReadLineExact("INITREMOTE-FAILURE this rclone remote does not support empty directories") -} diff --git a/cmd/gitannex/layout.go b/cmd/gitannex/layout.go index 81b18bfb6..897cdb5e0 100644 --- a/cmd/gitannex/layout.go +++ b/cmd/gitannex/layout.go @@ -3,6 +3,8 @@ package gitannex import ( "fmt" "strings" + + "github.com/rclone/rclone/fs/fspath" ) type layoutMode string @@ -39,8 +41,11 @@ func parseLayoutMode(mode string) layoutMode { type queryDirhashFunc func(msg string) (string, error) func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteName, prefix string) (string, error) { + remoteName = strings.TrimSuffix(remoteName, ":") + ":" + remoteString := fspath.JoinRootPath(remoteName, prefix) + if mode == layoutModeNodir { - return fmt.Sprintf("%s:%s", remoteName, prefix), nil + return remoteString, nil } var dirhash string @@ -59,13 +64,13 @@ func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteNa switch mode { case layoutModeLower: - return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil + return fmt.Sprintf("%s/%s", remoteString, dirhash), nil case layoutModeDirectory: - return fmt.Sprintf("%s:%s/%s%s", remoteName, prefix, dirhash, key), nil + return fmt.Sprintf("%s/%s%s", remoteString, dirhash, key), nil case layoutModeMixed: - return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil + return fmt.Sprintf("%s/%s", remoteString, dirhash), nil case layoutModeFrankencase: - return fmt.Sprintf("%s:%s/%s", remoteName, prefix, strings.ToLower(dirhash)), nil + return fmt.Sprintf("%s/%s", remoteString, strings.ToLower(dirhash)), nil default: panic("unreachable") }