From dfc329c03655e3ab9a51da83456a688a436c4f50 Mon Sep 17 00:00:00 2001 From: Dan McArdle Date: Sun, 28 Jan 2024 13:36:17 -0500 Subject: [PATCH] cmd/gitannex: Add the gitannex subcommand This commit adds a new subcommand named "gitannex", aka "git-annex-remote-rclone-builtin" when invoked via a symlink. This accomplishes milestone 1 from issue #7625: "minimal support for the external special remote protocol". Issue #7625 --- cmd/all/all.go | 1 + cmd/gitannex/gitannex.go | 515 ++++++++++++ cmd/gitannex/gitannex.md | 38 + cmd/gitannex/gitannex_test.go | 969 +++++++++++++++++++++++ cmd/gitannex/run-git-annex-testremote.sh | 52 ++ 5 files changed, 1575 insertions(+) create mode 100644 cmd/gitannex/gitannex.go create mode 100644 cmd/gitannex/gitannex.md create mode 100644 cmd/gitannex/gitannex_test.go create mode 100755 cmd/gitannex/run-git-annex-testremote.sh diff --git a/cmd/all/all.go b/cmd/all/all.go index 5fb87ed16..37b15c5c3 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -25,6 +25,7 @@ import ( _ "github.com/rclone/rclone/cmd/deletefile" _ "github.com/rclone/rclone/cmd/genautocomplete" _ "github.com/rclone/rclone/cmd/gendocs" + _ "github.com/rclone/rclone/cmd/gitannex" _ "github.com/rclone/rclone/cmd/hashsum" _ "github.com/rclone/rclone/cmd/link" _ "github.com/rclone/rclone/cmd/listremotes" diff --git a/cmd/gitannex/gitannex.go b/cmd/gitannex/gitannex.go new file mode 100644 index 000000000..0c085c133 --- /dev/null +++ b/cmd/gitannex/gitannex.go @@ -0,0 +1,515 @@ +// Package gitannex provides the "gitannex" command, which enables [git-annex] +// to communicate with rclone by implementing the [external special remote +// protocol]. The protocol is line delimited and spoken over stdin and stdout. +// +// # Milestones +// +// (Tracked in [issue #7625].) +// +// 1. ✅ Minimal support for the [external special remote protocol]. Tested on +// "local" and "drive" backends. +// 2. Add support for the ASYNC protocol extension. This may improve performance. +// 3. Support the [simple export interface]. This will enable `git-annex +// export` functionality. +// 4. Once the draft is finalized, support import/export interface. +// +// [git-annex]: https://git-annex.branchable.com/ +// [external special remote protocol]: https://git-annex.branchable.com/design/external_special_remote_protocol/ +// [simple export interface]: https://git-annex.branchable.com/design/external_special_remote_protocol/export_and_import_appendix/ +// [issue #7625]: https://github.com/rclone/rclone/issues/7625 +package gitannex + +import ( + "bufio" + "context" + _ "embed" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/operations" + "github.com/spf13/cobra" +) + +const subcommandName string = "gitannex" +const uniqueCommandName string = "git-annex-remote-rclone-builtin" + +//go:embed gitannex.md +var gitannexHelp string + +func init() { + os.Args = maybeTransformArgs(os.Args) + cmd.Root.AddCommand(command) +} + +// maybeTransformArgs returns a modified version of `args` with the "gitannex" +// subcommand inserted when `args` indicates that the program was executed as +// "git-annex-remote-rclone-builtin". One way this can happen is when rclone is +// invoked via symlink. Otherwise, returns `args`. +func maybeTransformArgs(args []string) []string { + if len(args) == 0 || filepath.Base(args[0]) != uniqueCommandName { + return args + } + newArgs := make([]string, 0, len(args)+1) + newArgs = append(newArgs, args[0]) + newArgs = append(newArgs, subcommandName) + newArgs = append(newArgs, args[1:]...) + return newArgs +} + +// messageParser helps parse messages we receive from git-annex into a sequence +// of parameters. Messages are not quite trivial to parse because they are +// separated by spaces, but the final parameter may itself contain spaces. +// +// This abstraction is necessary because simply splitting on space doesn't cut +// it. Also, we cannot know how many parameters to parse until we've parsed the +// first parameter. +type messageParser struct { + line string +} + +// nextSpaceDelimitedParameter consumes the next space-delimited parameter. +func (m *messageParser) nextSpaceDelimitedParameter() (string, error) { + m.line = strings.TrimRight(m.line, "\r\n") + if len(m.line) == 0 { + return "", errors.New("nothing remains to parse") + } + + before, after, found := strings.Cut(m.line, " ") + if found { + if len(before) == 0 { + return "", fmt.Errorf("found an empty space-delimited parameter in line: %q", m.line) + } + m.line = after + return before, nil + } + + remaining := m.line + m.line = "" + return remaining, nil +} + +// finalParameter consumes the final parameter, which may contain spaces. +func (m *messageParser) finalParameter() (string, error) { + m.line = strings.TrimRight(m.line, "\r\n") + if len(m.line) == 0 { + return "", errors.New("nothing remains to parse") + } + + param := m.line + m.line = "" + return param, nil +} + +// configDefinition describes a configuration value required by this command. We +// use "GETCONFIG" messages to query git-annex for these values at runtime. +type configDefinition struct { + name string + description string + destination *string +} + +// server contains this command's current state. +type server struct { + reader *bufio.Reader + writer io.Writer + + // When true, the server prints a transcript of messages sent and received + // to stderr. + verbose bool + + extensionInfo bool + extensionAsync bool + extensionGetGitRemoteName bool + extensionUnavailableResponse bool + + configsDone bool + configPrefix string + configRcloneRemoteName string +} + +func (s *server) sendMsg(msg string) { + msg = msg + "\n" + if _, err := io.WriteString(s.writer, msg); err != nil { + panic(err) + } + if s.verbose { + _, err := os.Stderr.WriteString(fmt.Sprintf("server sent %q\n", msg)) + if err != nil { + panic(fmt.Errorf("failed to write verbose message to stderr: %w", err)) + } + } +} + +func (s *server) getMsg() (*messageParser, error) { + msg, err := s.reader.ReadString('\n') + if err != nil { + if len(msg) == 0 { + // Git-annex closes stdin when it is done with us, so failing to + // read a new line is not an error. + return nil, nil + } + return nil, fmt.Errorf("expected message to end with newline: %q", msg) + } + if s.verbose { + _, err := os.Stderr.WriteString(fmt.Sprintf("server received %q\n", msg)) + if err != nil { + return nil, fmt.Errorf("failed to write verbose message to stderr: %w", err) + } + } + return &messageParser{msg}, nil +} + +func (s *server) run() error { + // The remote sends the first message. + s.sendMsg("VERSION 2") + + for { + message, err := s.getMsg() + if err != nil { + return fmt.Errorf("error receiving message: %w", err) + } + + if message == nil { + break + } + + command, err := message.nextSpaceDelimitedParameter() + if err != nil { + return fmt.Errorf("failed to parse command") + } + + switch command { + // + // Git-annex requires that these requests are supported. + // + case "INITREMOTE": + err = s.handleInitRemote() + case "PREPARE": + err = s.handlePrepare() + case "EXPORTSUPPORTED": + // Indicate that we do not support exports. + s.sendMsg("EXPORTSUPPORTED-FAILURE") + case "TRANSFER": + err = s.handleTransfer(message) + case "CHECKPRESENT": + err = s.handleCheckPresent(message) + case "REMOVE": + err = s.handleRemove(message) + case "ERROR": + errorMessage, parseErr := message.finalParameter() + if parseErr != nil { + err = fmt.Errorf("error while parsing ERROR message from git-annex: %w", parseErr) + break + } + err = fmt.Errorf("received error message from git-annex: %s", errorMessage) + + // + // These requests are optional. + // + case "EXTENSIONS": + // Git-annex just told us which protocol extensions it supports. + // Respond with the list of extensions that we want to use (none). + err = s.handleExtensions(message) + case "LISTCONFIGS": + s.handleListConfigs() + case "GETCOST": + // Git-annex wants to know the "cost" of using this remote. It + // probably depends on the backend we will be using, but let's just + // consider this an "expensive remote" per git-annex's + // Config/Cost.hs. + s.sendMsg("COST 200") + case "GETAVAILABILITY": + // Indicate that this is a cloud service. + s.sendMsg("AVAILABILITY GLOBAL") + case "CLAIMURL", "CHECKURL", "WHEREIS", "GETINFO": + s.sendMsg("UNSUPPORTED-REQUEST") + default: + err = fmt.Errorf("received unexpected message from git-annex: %s", message.line) + } + if err != nil { + return err + } + } + + return nil +} + +// Idempotently handle an incoming INITREMOTE message. This should perform +// one-time setup operations, but we may receive the command again, e.g. when +// this git-annex remote is initialized in a different repository. +func (s *server) handleInitRemote() error { + if err := s.queryConfigs(); err != nil { + return fmt.Errorf("failed to get configs: %w", err) + } + + remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName)) + if err != nil { + s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote") + return fmt.Errorf("failed to open root directory of rclone remote: %w", err) + } + + if !remoteRootFs.Features().CanHaveEmptyDirectories { + s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories") + return fmt.Errorf("rclone remote does not support empty directories") + } + + if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil { + s.sendMsg("INITREMOTE-FAILURE failed to mkdir") + return fmt.Errorf("failed to mkdir: %w", err) + } + + s.sendMsg("INITREMOTE-SUCCESS") + return nil +} + +// Get a list of configs with pointers to fields of `s`. +func (s *server) getRequiredConfigs() []configDefinition { + return []configDefinition{ + { + "rcloneremotename", + "Name of the rclone remote to use. " + + "Must match a remote known to rclone. " + + "(Note that rclone remotes are a distinct concept from git-annex remotes.)", + &s.configRcloneRemoteName, + }, + { + "rcloneprefix", + "Directory where rclone will write git-annex content. " + + "If not specified, defaults to \"git-annex-rclone\". " + + "This directory be created on init if it does not exist.", + &s.configPrefix, + }, + } +} + +// Query git-annex for config values. +func (s *server) queryConfigs() error { + if s.configsDone { + return nil + } + + // Send a "GETCONFIG" message for each required config and parse git-annex's + // "VALUE" response. + for _, config := range s.getRequiredConfigs() { + s.sendMsg(fmt.Sprintf("GETCONFIG %s", config.name)) + + message, err := s.getMsg() + if err != nil { + return err + } + + valueKeyword, err := message.nextSpaceDelimitedParameter() + if err != nil || valueKeyword != "VALUE" { + return fmt.Errorf("failed to parse config value: %s %s", valueKeyword, message.line) + } + + value, err := message.finalParameter() + if err != nil || value == "" { + return fmt.Errorf("config value of %q must not be empty", config.name) + } + + *config.destination = value + } + + s.configsDone = true + return nil +} + +func (s *server) handlePrepare() error { + if err := s.queryConfigs(); err != nil { + s.sendMsg("PREPARE-FAILURE Error getting configs") + return fmt.Errorf("error getting configs: %w", err) + } + s.sendMsg("PREPARE-SUCCESS") + return nil +} + +// Git-annex is asking us to return the list of settings that we use. Keep this +// in sync with `handlePrepare()`. +func (s *server) handleListConfigs() { + for _, config := range s.getRequiredConfigs() { + s.sendMsg(fmt.Sprintf("CONFIG %s %s", config.name, config.description)) + } + s.sendMsg("CONFIGEND") +} + +func (s *server) handleTransfer(message *messageParser) error { + argMode, err := message.nextSpaceDelimitedParameter() + if err != nil { + s.sendMsg("TRANSFER-FAILURE failed to parse direction") + return fmt.Errorf("malformed arguments for TRANSFER: %w", err) + } + argKey, err := message.nextSpaceDelimitedParameter() + if err != nil { + s.sendMsg("TRANSFER-FAILURE failed to parse key") + return fmt.Errorf("malformed arguments for TRANSFER: %w", err) + } + argFile, err := message.finalParameter() + if err != nil { + s.sendMsg("TRANSFER-FAILURE failed to parse file") + return fmt.Errorf("malformed arguments for TRANSFER: %w", err) + } + + if err := s.queryConfigs(); err != nil { + s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get configs", argMode, argKey)) + return fmt.Errorf("error getting configs: %w", err) + } + + remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix)) + if err != nil { + s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey)) + return err + } + + localDir := filepath.Dir(argFile) + localFs, err := cache.Get(context.TODO(), localDir) + if err != nil { + s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get local fs", argMode, argKey)) + return fmt.Errorf("failed to get local fs: %w", err) + } + + remoteFileName := argKey + localFileName := filepath.Base(argFile) + + switch argMode { + case "STORE": + err = operations.CopyFile(context.TODO(), remoteFs, localFs, remoteFileName, localFileName) + if err != nil { + s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err)) + return err + } + + case "RETRIEVE": + err = operations.CopyFile(context.TODO(), localFs, remoteFs, localFileName, remoteFileName) + // It is non-fatal when retrieval fails because the file is missing on + // the remote. + if err == fs.ErrorObjectNotFound { + s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s not found", argMode, argKey)) + return nil + } + if err != nil { + s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err)) + return err + } + + default: + s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s unrecognized mode", argMode, argKey)) + return fmt.Errorf("received malformed TRANSFER mode: %v", argMode) + } + + s.sendMsg(fmt.Sprintf("TRANSFER-SUCCESS %s %s", argMode, argKey)) + return nil +} + +func (s *server) handleCheckPresent(message *messageParser) error { + argKey, err := message.finalParameter() + if err != nil { + return err + } + + if err := s.queryConfigs(); err != nil { + s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s failed to get configs", argKey)) + return fmt.Errorf("error getting configs: %s", err) + } + + remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix)) + if err != nil { + s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey)) + return err + } + + _, err = remoteFs.NewObject(context.TODO(), argKey) + if err == fs.ErrorObjectNotFound { + s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey)) + return nil + } + if err != nil { + s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s error finding file", argKey)) + return err + } + + s.sendMsg(fmt.Sprintf("CHECKPRESENT-SUCCESS %s", argKey)) + return nil +} + +func (s *server) handleRemove(message *messageParser) error { + argKey, err := message.finalParameter() + if err != nil { + return err + } + + remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix)) + if err != nil { + s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey)) + return fmt.Errorf("error getting remote fs: %w", err) + } + + fileObj, err := remoteFs.NewObject(context.TODO(), argKey) + // It is non-fatal when removal fails because the file is missing on the + // remote. + if errors.Is(err, fs.ErrorObjectNotFound) { + s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey)) + return nil + } + if err != nil { + s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error getting new fs object: %s", argKey, err)) + return fmt.Errorf("error getting new fs object: %w", err) + } + if err := operations.DeleteFile(context.TODO(), fileObj); err != nil { + s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error deleting file", argKey)) + return fmt.Errorf("error deleting file: %q", argKey) + } + s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey)) + return nil +} + +func (s *server) handleExtensions(message *messageParser) error { + for { + extension, err := message.nextSpaceDelimitedParameter() + if err != nil { + break + } + switch extension { + case "INFO": + s.extensionInfo = true + case "ASYNC": + s.extensionAsync = true + case "GETGITREMOTENAME": + s.extensionGetGitRemoteName = true + case "UNAVAILABLERESPONSE": + s.extensionUnavailableResponse = true + } + } + s.sendMsg("EXTENSIONS") + return nil +} + +var command = &cobra.Command{ + Aliases: []string{uniqueCommandName}, + Use: subcommandName, + Short: "Speaks with git-annex over stdin/stdout.", + Long: gitannexHelp, + Annotations: map[string]string{ + "versionIntroduced": "v1.67.0", + }, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(0, 0, command, args) + + s := server{ + reader: bufio.NewReader(os.Stdin), + writer: os.Stdout, + } + err := s.run() + if err != nil { + s.sendMsg(fmt.Sprintf("ERROR %s", err.Error())) + panic(err) + } + }, +} diff --git a/cmd/gitannex/gitannex.md b/cmd/gitannex/gitannex.md new file mode 100644 index 000000000..c36affffe --- /dev/null +++ b/cmd/gitannex/gitannex.md @@ -0,0 +1,38 @@ +Rclone's gitannex subcommand enables git-annex to store and retrieve content +from an rclone remote. It expects to be run by git-annex, not directly by users. +It is an "external special remote program" as defined by git-annex. + +Installation on Linux +--------------------- + +1. Create a symlink and ensure it's on your PATH. For example: + + ln -s "$(realpath rclone)" "$HOME/bin/git-annex-remote-rclone-builtin" + +2. Add a new external remote to your git-annex repo. + + The new remote's type should be "rclone-builtin". When git-annex interacts + with remotes of this type, it will try to run a command named + "git-annex-remote-rclone-builtin", so the symlink from the previous step + should be on your PATH. + + The following example creates a new git-annex remote named "MyRemote" that + will use the rclone remote named "SomeRcloneRemote". This rclone remote must + be configured in your rclone.conf file, wherever that is located on your + system. The rcloneprefix value ensures that content is only written into the + rclone remote underneath the "git-annex-content" directory. + + git annex initremote MyRemote \ + type=external \ + externaltype=rclone-builtin \ + encryption=none \ + rcloneremotename=SomeRcloneRemote \ + rcloneprefix=git-annex-content + +3. Before you trust this command with your precious data, be sure to **test the + remote**. This command is very new and has not been tested on many rclone + backends. Caveat emptor! + + git annex testremote my-rclone-remote + +Happy annexing! diff --git a/cmd/gitannex/gitannex_test.go b/cmd/gitannex/gitannex_test.go new file mode 100644 index 000000000..5581d400b --- /dev/null +++ b/cmd/gitannex/gitannex_test.go @@ -0,0 +1,969 @@ +package gitannex + +import ( + "bufio" + "crypto/sha256" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + // Without this import, the local filesystem backend would be unavailable. + // It looks unused, but the act of importing it runs its `init()` function. + _ "github.com/rclone/rclone/backend/local" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/config/configfile" + "github.com/rclone/rclone/fstest/mockfs" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFixArgsForSymlinkIdentity(t *testing.T) { + for _, argList := range [][]string{ + []string{}, + []string{"foo"}, + []string{"foo", "bar"}, + []string{"foo", "bar", "baz"}, + } { + assert.Equal(t, maybeTransformArgs(argList), argList) + } +} + +func TestFixArgsForSymlinkCorrectName(t *testing.T) { + assert.Equal(t, + maybeTransformArgs([]string{"git-annex-remote-rclone-builtin"}), + []string{"git-annex-remote-rclone-builtin", "gitannex"}) + assert.Equal(t, + maybeTransformArgs([]string{"/path/to/git-annex-remote-rclone-builtin"}), + []string{"/path/to/git-annex-remote-rclone-builtin", "gitannex"}) +} + +type messageParserTestCase struct { + label string + testFunc func(*testing.T) +} + +var messageParserTestCases = []messageParserTestCase{ + { + "OneParam", + func(t *testing.T) { + m := messageParser{"foo\n"} + + param, err := m.nextSpaceDelimitedParameter() + assert.NoError(t, err) + assert.Equal(t, param, "foo") + + param, err = m.nextSpaceDelimitedParameter() + assert.Error(t, err) + assert.Equal(t, param, "") + + param, err = m.finalParameter() + assert.Error(t, err) + assert.Equal(t, param, "") + + param, err = m.finalParameter() + assert.Error(t, err) + assert.Equal(t, param, "") + + param, err = m.nextSpaceDelimitedParameter() + assert.Error(t, err) + assert.Equal(t, param, "") + + }, + }, + { + "TwoParams", + func(t *testing.T) { + m := messageParser{"foo bar\n"} + + param, err := m.nextSpaceDelimitedParameter() + assert.NoError(t, err) + assert.Equal(t, param, "foo") + + param, err = m.nextSpaceDelimitedParameter() + assert.NoError(t, err) + assert.Equal(t, param, "bar") + + param, err = m.nextSpaceDelimitedParameter() + assert.Error(t, err) + assert.Equal(t, param, "") + + param, err = m.finalParameter() + assert.Error(t, err) + assert.Equal(t, param, "") + }, + }, + { + "TwoParamsNoTrailingNewline", + + func(t *testing.T) { + m := messageParser{"foo bar"} + + param, err := m.nextSpaceDelimitedParameter() + assert.NoError(t, err) + assert.Equal(t, param, "foo") + + param, err = m.nextSpaceDelimitedParameter() + assert.NoError(t, err) + assert.Equal(t, param, "bar") + + param, err = m.nextSpaceDelimitedParameter() + assert.Error(t, err) + assert.Equal(t, param, "") + + param, err = m.finalParameter() + assert.Error(t, err) + assert.Equal(t, param, "") + }, + }, + { + "ThreeParamsWhereFinalParamContainsSpaces", + func(t *testing.T) { + m := messageParser{"firstparam secondparam final param with spaces"} + + param, err := m.nextSpaceDelimitedParameter() + assert.NoError(t, err) + assert.Equal(t, param, "firstparam") + + param, err = m.nextSpaceDelimitedParameter() + assert.NoError(t, err) + assert.Equal(t, param, "secondparam") + + param, err = m.finalParameter() + assert.NoError(t, err) + assert.Equal(t, param, "final param with spaces") + }, + }, + { + "OneLongFinalParameter", + func(t *testing.T) { + for _, lineEnding := range []string{"", "\n", "\r", "\r\n", "\n\r"} { + lineEnding := lineEnding + testName := fmt.Sprintf("lineEnding%x", lineEnding) + + t.Run(testName, func(t *testing.T) { + m := messageParser{"one long final parameter" + lineEnding} + + param, err := m.finalParameter() + assert.NoError(t, err) + assert.Equal(t, param, "one long final parameter") + + param, err = m.finalParameter() + assert.Error(t, err) + assert.Equal(t, param, "") + }) + + } + }, + }, + { + "MultipleSpaces", + func(t *testing.T) { + m := messageParser{"foo bar\n\r"} + + param, err := m.nextSpaceDelimitedParameter() + assert.NoError(t, err) + assert.Equal(t, param, "foo") + + param, err = m.nextSpaceDelimitedParameter() + assert.Error(t, err, "blah") + assert.Equal(t, param, "") + }, + }, + { + "StartsWithSpace", + func(t *testing.T) { + m := messageParser{" foo"} + + param, err := m.nextSpaceDelimitedParameter() + assert.Error(t, err, "blah") + assert.Equal(t, param, "") + }, + }, +} + +func TestMessageParser(t *testing.T) { + for _, testCase := range messageParserTestCases { + testCase := testCase + t.Run(testCase.label, func(t *testing.T) { + t.Parallel() + testCase.testFunc(t) + }) + } +} + +type testState struct { + t *testing.T + server *server + mockStdinW *io.PipeWriter + mockStdoutReader *bufio.Reader + + localFsDir string + configPath string + remoteName string +} + +func makeTestState(t *testing.T) testState { + stdinR, stdinW := io.Pipe() + stdoutR, stdoutW := io.Pipe() + + return testState{ + t: t, + server: &server{ + reader: bufio.NewReader(stdinR), + writer: stdoutW, + }, + mockStdinW: stdinW, + mockStdoutReader: bufio.NewReader(stdoutR), + } +} + +func (h *testState) requireReadLineExact(line string) { + receivedLine, err := h.mockStdoutReader.ReadString('\n') + require.NoError(h.t, err) + require.Equal(h.t, line+"\n", receivedLine) +} + +func (h *testState) requireWriteLine(line string) { + _, err := h.mockStdinW.Write([]byte(line + "\n")) + require.NoError(h.t, err) +} + +// Preconfigure the handle. This enables the calling test to skip the PREPARE +// handshake. +func (h *testState) preconfigureServer() { + h.server.configPrefix = h.localFsDir + h.server.configRcloneRemoteName = h.remoteName + h.server.configsDone = true +} + +// getUniqueRemoteName returns a valid remote name derived from the given test's +// name. This is necessary because when a test registers a second remote with +// the same name, the original remote appears to take precedence. This function +// is injective, so each test gets a unique remote name. Returned strings +// contain no spaces. +func getUniqueRemoteName(t *testing.T) string { + // Using sha256 as a hack to ensure injectivity without adding a global + // variable. + return fmt.Sprintf("remote-%x", sha256.Sum256([]byte(t.Name()))) +} + +type testCase struct { + label string + testProtocolFunc func(*testing.T, *testState) + expectedError string +} + +// These test cases run against the "local" backend. +var localBackendTestCases = []testCase{ + { + label: "HandlesInit", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "HandlesPrepare", + testProtocolFunc: func(t *testing.T, h *testState) { + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension + h.requireReadLineExact("EXTENSIONS") + + if !h.server.extensionInfo { + t.Errorf("expected INFO extension to be enabled") + return + } + + h.requireWriteLine("PREPARE") + h.requireReadLineExact("GETCONFIG rcloneremotename") + h.requireWriteLine("VALUE " + h.remoteName) + h.requireReadLineExact("GETCONFIG rcloneprefix") + h.requireWriteLine("VALUE " + h.localFsDir) + h.requireReadLineExact("PREPARE-SUCCESS") + + require.Equal(t, h.server.configRcloneRemoteName, h.remoteName) + require.Equal(t, h.server.configPrefix, h.localFsDir) + require.True(t, h.server.configsDone) + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "HandlesPrepareAndDoesNotTrimWhitespaceFromValue", + testProtocolFunc: func(t *testing.T, h *testState) { + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension + h.requireReadLineExact("EXTENSIONS") + + if !h.server.extensionInfo { + t.Errorf("expected INFO extension to be enabled") + return + } + + h.requireWriteLine("PREPARE") + h.requireReadLineExact("GETCONFIG rcloneremotename") + + remoteNameWithSpaces := fmt.Sprintf(" %s ", h.remoteName) + localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir) + + h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces)) + h.requireReadLineExact("GETCONFIG rcloneprefix") + + h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces)) + h.requireReadLineExact("PREPARE-SUCCESS") + + require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces) + require.Equal(t, h.server.configPrefix, localFsDirWithSpaces) + require.True(t, h.server.configsDone) + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "HandlesEarlyError", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("ERROR foo") + + require.NoError(t, h.mockStdinW.Close()) + }, + expectedError: "received error message from git-annex: foo", + }, + // Test what happens when the git-annex client sends "GETCONFIG", but + // doesn't understand git-annex's response. + { + label: "ConfigFail", + testProtocolFunc: func(t *testing.T, h *testState) { + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension + h.requireReadLineExact("EXTENSIONS") + require.True(t, h.server.extensionInfo) + + h.requireWriteLine("PREPARE") + h.requireReadLineExact("GETCONFIG rcloneremotename") + h.requireWriteLine("ERROR ineffable error") + h.requireReadLineExact("PREPARE-FAILURE Error getting configs") + + require.NoError(t, h.mockStdinW.Close()) + }, + expectedError: "failed to parse config value: ERROR ineffable error", + }, + { + label: "TransferStoreEmptyPath", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + // Note the whitespace following the key. + h.requireWriteLine("TRANSFER STORE Key ") + h.requireReadLineExact("TRANSFER-FAILURE failed to parse file") + + require.NoError(t, h.mockStdinW.Close()) + }, + expectedError: "malformed arguments for TRANSFER: nothing remains to parse", + }, + // Repeated EXTENSIONS messages add to each other rather than overriding + // prior advertised extensions. This behavior is not mandated by the + // protocol design. + { + label: "ExtensionsCompound", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + h.requireWriteLine("EXTENSIONS") + h.requireReadLineExact("EXTENSIONS") + require.False(t, h.server.extensionInfo) + require.False(t, h.server.extensionAsync) + require.False(t, h.server.extensionGetGitRemoteName) + require.False(t, h.server.extensionUnavailableResponse) + + h.requireWriteLine("EXTENSIONS INFO") + h.requireReadLineExact("EXTENSIONS") + require.True(t, h.server.extensionInfo) + require.False(t, h.server.extensionAsync) + require.False(t, h.server.extensionGetGitRemoteName) + require.False(t, h.server.extensionUnavailableResponse) + + h.requireWriteLine("EXTENSIONS ASYNC") + h.requireReadLineExact("EXTENSIONS") + require.True(t, h.server.extensionInfo) + require.True(t, h.server.extensionAsync) + require.False(t, h.server.extensionGetGitRemoteName) + require.False(t, h.server.extensionUnavailableResponse) + + h.requireWriteLine("EXTENSIONS GETGITREMOTENAME") + h.requireReadLineExact("EXTENSIONS") + require.True(t, h.server.extensionInfo) + require.True(t, h.server.extensionAsync) + require.True(t, h.server.extensionGetGitRemoteName) + require.False(t, h.server.extensionUnavailableResponse) + + h.requireWriteLine("EXTENSIONS UNAVAILABLERESPONSE") + h.requireReadLineExact("EXTENSIONS") + require.True(t, h.server.extensionInfo) + require.True(t, h.server.extensionAsync) + require.True(t, h.server.extensionGetGitRemoteName) + require.True(t, h.server.extensionUnavailableResponse) + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "ExtensionsIdempotent", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + h.requireWriteLine("EXTENSIONS") + h.requireReadLineExact("EXTENSIONS") + require.False(t, h.server.extensionInfo) + require.False(t, h.server.extensionAsync) + require.False(t, h.server.extensionGetGitRemoteName) + require.False(t, h.server.extensionUnavailableResponse) + + h.requireWriteLine("EXTENSIONS") + h.requireReadLineExact("EXTENSIONS") + require.False(t, h.server.extensionInfo) + require.False(t, h.server.extensionAsync) + require.False(t, h.server.extensionGetGitRemoteName) + require.False(t, h.server.extensionUnavailableResponse) + + h.requireWriteLine("EXTENSIONS INFO") + h.requireReadLineExact("EXTENSIONS") + require.True(t, h.server.extensionInfo) + require.False(t, h.server.extensionAsync) + require.False(t, h.server.extensionGetGitRemoteName) + require.False(t, h.server.extensionUnavailableResponse) + + h.requireWriteLine("EXTENSIONS INFO") + h.requireReadLineExact("EXTENSIONS") + require.True(t, h.server.extensionInfo) + require.False(t, h.server.extensionAsync) + require.False(t, h.server.extensionGetGitRemoteName) + require.False(t, h.server.extensionUnavailableResponse) + + h.requireWriteLine("EXTENSIONS ASYNC ASYNC") + h.requireReadLineExact("EXTENSIONS") + require.True(t, h.server.extensionInfo) + require.True(t, h.server.extensionAsync) + require.False(t, h.server.extensionGetGitRemoteName) + require.False(t, h.server.extensionUnavailableResponse) + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "ExtensionsSupportsMultiple", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + h.requireWriteLine("EXTENSIONS") + h.requireReadLineExact("EXTENSIONS") + require.False(t, h.server.extensionInfo) + require.False(t, h.server.extensionAsync) + require.False(t, h.server.extensionGetGitRemoteName) + require.False(t, h.server.extensionUnavailableResponse) + + h.requireWriteLine("EXTENSIONS INFO ASYNC") + h.requireReadLineExact("EXTENSIONS") + require.True(t, h.server.extensionInfo) + require.True(t, h.server.extensionAsync) + require.False(t, h.server.extensionGetGitRemoteName) + require.False(t, h.server.extensionUnavailableResponse) + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "TransferStoreAbsolute", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + // Create temp file for transfer with an absolute path. + fileToTransfer := filepath.Join(t.TempDir(), "file.txt") + require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + require.FileExists(t, fileToTransfer) + require.True(t, filepath.IsAbs(fileToTransfer)) + + // Specify an absolute path to transfer. + h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer) + h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute") + require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute")) + + // Transfer the same absolute path a second time, but with a different key. + h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + fileToTransfer) + h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute2") + require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute2")) + + h.requireWriteLine("CHECKPRESENT KeyAbsolute2") + h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyAbsolute2") + + h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist") + h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist") + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + // Test that the TRANSFER command understands simple relative paths + // consisting only of a file name. + { + label: "TransferStoreRelative", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + // Save the current working directory so we can restore it when this + // test ends. + cwd, err := os.Getwd() + require.NoError(t, err) + + require.NoError(t, os.Chdir(t.TempDir())) + t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) }) + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + // Create temp file for transfer with a relative path. + fileToTransfer := "file.txt" + require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + require.FileExists(t, fileToTransfer) + require.False(t, filepath.IsAbs(fileToTransfer)) + + // Specify a relative path to transfer. + h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer) + h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative") + require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative")) + + h.requireWriteLine("CHECKPRESENT KeyRelative") + h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative") + + h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist") + h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist") + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "TransferStorePathWithInteriorWhitespace", + testProtocolFunc: func(t *testing.T, h *testState) { + // Save the current working directory so we can restore it when this + // test ends. + cwd, err := os.Getwd() + require.NoError(t, err) + + require.NoError(t, os.Chdir(t.TempDir())) + t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) }) + + h.preconfigureServer() + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + // Create temp file for transfer. + fileToTransfer := "filename with spaces.txt" + require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + require.FileExists(t, fileToTransfer) + require.False(t, filepath.IsAbs(fileToTransfer)) + + // Specify a relative path to transfer. + h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer) + h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative") + require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative")) + + h.requireWriteLine("CHECKPRESENT KeyRelative") + h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative") + + h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist") + h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist") + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "CheckPresentAndTransfer", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + // Create temp file for transfer. + fileToTransfer := filepath.Join(t.TempDir(), "file.txt") + require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist") + h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist") + + // Specify an absolute path to transfer. + require.True(t, filepath.IsAbs(fileToTransfer)) + h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer) + h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute") + require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute")) + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + // Check whether a key is present, transfer a file with that key, then check + // again whether it is present. + // + // This is a regression test for a bug where the second CHECKPRESENT would + // generate the following response: + // + // CHECKPRESENT-UNKNOWN ${key} failed to read directory entry: readdirent ${filepath}: not a directory + // + // This message was generated by the local backend's `List()` function. When + // checking whether a file exists, we were erroneously listing its contents as + // if it were a directory. + { + label: "CheckpresentTransferCheckpresent", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + // Create temp file for transfer. + fileToTransfer := filepath.Join(t.TempDir(), "file.txt") + require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + h.requireWriteLine("CHECKPRESENT foo") + h.requireReadLineExact("CHECKPRESENT-FAILURE foo") + + h.requireWriteLine("TRANSFER STORE foo " + fileToTransfer) + h.requireReadLineExact("TRANSFER-SUCCESS STORE foo") + require.FileExists(t, filepath.Join(h.localFsDir, "foo")) + + h.requireWriteLine("CHECKPRESENT foo") + h.requireReadLineExact("CHECKPRESENT-SUCCESS foo") + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "TransferAndCheckpresentWithRealisticKey", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + // Create temp file for transfer. + fileToTransfer := filepath.Join(t.TempDir(), "file.txt") + require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + realisticKey := "SHA256E-s1048576--7ba87e06b9b7903cfbaf4a38736766c161e3e7b42f06fe57f040aa410a8f0701.this-is-a-test-key" + + // Specify an absolute path to transfer. + require.True(t, filepath.IsAbs(fileToTransfer)) + h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, fileToTransfer)) + h.requireReadLineExact("TRANSFER-SUCCESS STORE " + realisticKey) + require.FileExists(t, filepath.Join(h.localFsDir, realisticKey)) + + h.requireWriteLine("CHECKPRESENT " + realisticKey) + h.requireReadLineExact("CHECKPRESENT-SUCCESS " + realisticKey) + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "RetrieveNonexistentFile", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + h.requireWriteLine("TRANSFER RETRIEVE SomeKey path") + h.requireReadLineExact("TRANSFER-FAILURE RETRIEVE SomeKey not found") + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "StoreCheckpresentRetrieve", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + // Create temp file for transfer. + fileToTransfer := filepath.Join(t.TempDir(), "file.txt") + require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + // Specify an absolute path to transfer. + require.True(t, filepath.IsAbs(fileToTransfer)) + h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer) + h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey") + require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey")) + + h.requireWriteLine("CHECKPRESENT SomeKey") + h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey") + + retrievedFilePath := fileToTransfer + ".retrieved" + require.NoFileExists(t, retrievedFilePath) + h.requireWriteLine("TRANSFER RETRIEVE SomeKey " + retrievedFilePath) + h.requireReadLineExact("TRANSFER-SUCCESS RETRIEVE SomeKey") + require.FileExists(t, retrievedFilePath) + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "RemovePreexistingFile", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + // Write a file into the remote without using the git-annex + // protocol. + remoteFilePath := filepath.Join(h.localFsDir, "SomeKey") + require.NoError(t, os.WriteFile(remoteFilePath, []byte("HELLO"), 0600)) + require.FileExists(t, remoteFilePath) + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + h.requireWriteLine("CHECKPRESENT SomeKey") + h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey") + require.FileExists(t, remoteFilePath) + + h.requireWriteLine("REMOVE SomeKey") + h.requireReadLineExact("REMOVE-SUCCESS SomeKey") + require.NoFileExists(t, remoteFilePath) + + h.requireWriteLine("CHECKPRESENT SomeKey") + h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey") + require.NoFileExists(t, remoteFilePath) + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "Remove", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + // Create temp file for transfer. + fileToTransfer := filepath.Join(t.TempDir(), "file.txt") + require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + h.requireWriteLine("CHECKPRESENT SomeKey") + h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey") + + // Specify an absolute path to transfer. + require.True(t, filepath.IsAbs(fileToTransfer)) + h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer) + h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey") + require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey")) + + h.requireWriteLine("CHECKPRESENT SomeKey") + h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey") + + h.requireWriteLine("REMOVE SomeKey") + h.requireReadLineExact("REMOVE-SUCCESS SomeKey") + require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey")) + + h.requireWriteLine("CHECKPRESENT SomeKey") + h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey") + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "RemoveNonexistentFile", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + // Create temp file for transfer. + fileToTransfer := filepath.Join(t.TempDir(), "file.txt") + require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600)) + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + h.requireWriteLine("CHECKPRESENT SomeKey") + h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey") + + require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey")) + h.requireWriteLine("REMOVE SomeKey") + h.requireReadLineExact("REMOVE-SUCCESS SomeKey") + require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey")) + + h.requireWriteLine("CHECKPRESENT SomeKey") + h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey") + + require.NoError(t, h.mockStdinW.Close()) + }, + }, + { + label: "ExportNotSupported", + testProtocolFunc: func(t *testing.T, h *testState) { + h.preconfigureServer() + + h.requireReadLineExact("VERSION 2") + h.requireWriteLine("INITREMOTE") + h.requireReadLineExact("INITREMOTE-SUCCESS") + + h.requireWriteLine("EXPORTSUPPORTED") + h.requireReadLineExact("EXPORTSUPPORTED-FAILURE") + + require.NoError(t, h.mockStdinW.Close()) + }, + }, +} + +func TestGitAnnexLocalBackendCases(t *testing.T) { + for _, testCase := range localBackendTestCases { + // Clear global state left behind by tests that chdir to a temp directory. + cache.Clear() + + // TODO: Remove this when rclone requires a Go version >= 1.22. Future + // versions of Go fix the semantics of capturing a range variable. + // https://go.dev/blog/loopvar-preview + testCase := testCase + + t.Run(testCase.label, func(t *testing.T) { + tempDir := t.TempDir() + + // Create temp dir for an rclone remote pointing at local filesystem. + localFsDir := filepath.Join(tempDir, "remoteTarget") + require.NoError(t, os.Mkdir(localFsDir, 0700)) + + // Create temp config + remoteName := getUniqueRemoteName(t) + configLines := []string{ + fmt.Sprintf("[%s]", remoteName), + "type = local", + fmt.Sprintf("remote = %s", localFsDir), + } + configContents := strings.Join(configLines, "\n") + + configPath := filepath.Join(tempDir, "rclone.conf") + require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600)) + require.NoError(t, config.SetConfigPath(configPath)) + + // The custom config file will be ignored unless we install the + // global config file handler. + configfile.Install() + + handle := makeTestState(t) + handle.localFsDir = localFsDir + handle.configPath = configPath + handle.remoteName = remoteName + + var wg sync.WaitGroup + wg.Add(1) + + go func() { + err := handle.server.run() + + if testCase.expectedError == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, testCase.expectedError) + } + + wg.Done() + }() + defer wg.Wait() + + testCase.testProtocolFunc(t, &handle) + }) + } +} + +// Configure the git-annex client with a mockfs backend and send it the +// "INITREMOTE" command over mocked stdin. This should fail because mockfs does +// not support empty directories. +func TestGitAnnexHandleInitRemoteBackendDoesNotSupportEmptyDirectories(t *testing.T) { + tempDir := t.TempDir() + + // Temporarily override the filesystem registry. + oldRegistry := fs.Registry + mockfs.Register() + defer func() { fs.Registry = oldRegistry }() + + // Create temp dir for an rclone remote pointing at local filesystem. + localFsDir := filepath.Join(tempDir, "remoteTarget") + require.NoError(t, os.Mkdir(localFsDir, 0700)) + + // Create temp config + remoteName := getUniqueRemoteName(t) + configLines := []string{ + fmt.Sprintf("[%s]", remoteName), + "type = mockfs", + fmt.Sprintf("remote = %s", localFsDir), + } + configContents := strings.Join(configLines, "\n") + + configPath := filepath.Join(tempDir, "rclone.conf") + require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600)) + + // The custom config file will be ignored unless we install the global + // config file handler. + configfile.Install() + require.NoError(t, config.SetConfigPath(configPath)) + + handle := makeTestState(t) + handle.server.configPrefix = localFsDir + handle.server.configRcloneRemoteName = remoteName + handle.server.configsDone = true + + var wg sync.WaitGroup + wg.Add(1) + + go func() { + require.NotNil(t, handle.server.run()) + wg.Done() + }() + defer wg.Wait() + + handle.requireReadLineExact("VERSION 2") + handle.requireWriteLine("INITREMOTE") + handle.requireReadLineExact("INITREMOTE-FAILURE this rclone remote does not support empty directories") +} diff --git a/cmd/gitannex/run-git-annex-testremote.sh b/cmd/gitannex/run-git-annex-testremote.sh new file mode 100755 index 000000000..98a848df7 --- /dev/null +++ b/cmd/gitannex/run-git-annex-testremote.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# End-to-end tests for "rclone gitannex". This script runs the `git-annex +# testremote` suite against "rclone gitannex" in an ephemeral git-annex repo. +# +# Assumptions: +# +# * This system has an rclone remote configured named "git-annex-builtin-test-remote". +# +# * If it uses rclone's "local" backend, /tmp/git-annex-builtin-test-remote exists. + +set -e + +TEST_DIR="$(realpath "$(mktemp -d)")" +mkdir "$TEST_DIR/bin" + +function cleanup() +{ + rm -rf "$TEST_DIR" +} + +trap cleanup EXIT + +RCLONE_DIR="$(git rev-parse --show-toplevel)" + +rm -rf /tmp/git-annex-builtin-test-remote/* + +set -x + +pushd "$RCLONE_DIR" +go build -o "$TEST_DIR/bin" ./ + +ln -s "$(realpath "$TEST_DIR/bin/rclone")" "$TEST_DIR/bin/git-annex-remote-rclone-builtin" +popd + +pushd "$TEST_DIR" + +git init +git annex init + +REMOTE_NAME=git-annex-builtin-test-remote +PREFIX=/tmp/git-annex-builtin-test-remote + +PATH="$PATH:$TEST_DIR/bin" git annex initremote $REMOTE_NAME \ + type=external externaltype=rclone-builtin encryption=none \ + rcloneremotename=$REMOTE_NAME \ + rcloneprefix="$PREFIX" + +PATH="$PATH:$(realpath bin)" git annex testremote $REMOTE_NAME + +popd +rm -rf "$TEST_DIR"