mirror of
https://github.com/rclone/rclone.git
synced 2024-11-24 17:34:57 +01:00
cmd/gitannex: Add the gitannex subcommand
This commit adds a new subcommand named "gitannex", aka "git-annex-remote-rclone-builtin" when invoked via a symlink. This accomplishes milestone 1 from issue #7625: "minimal support for the external special remote protocol". Issue #7625
This commit is contained in:
Dan McArdle
Nick Craig-Wood
parent
d9601c78b1
commit
dfc329c036
@ -25,6 +25,7 @@ import (
|
||||
_ "github.com/rclone/rclone/cmd/deletefile"
|
||||
_ "github.com/rclone/rclone/cmd/genautocomplete"
|
||||
_ "github.com/rclone/rclone/cmd/gendocs"
|
||||
_ "github.com/rclone/rclone/cmd/gitannex"
|
||||
_ "github.com/rclone/rclone/cmd/hashsum"
|
||||
_ "github.com/rclone/rclone/cmd/link"
|
||||
_ "github.com/rclone/rclone/cmd/listremotes"
|
||||
|
||||
515
cmd/gitannex/gitannex.go
Normal file
515
cmd/gitannex/gitannex.go
Normal file
@ -0,0 +1,515 @@
|
||||
// Package gitannex provides the "gitannex" command, which enables [git-annex]
|
||||
// to communicate with rclone by implementing the [external special remote
|
||||
// protocol]. The protocol is line delimited and spoken over stdin and stdout.
|
||||
//
|
||||
// # Milestones
|
||||
//
|
||||
// (Tracked in [issue #7625].)
|
||||
//
|
||||
// 1. ✅ Minimal support for the [external special remote protocol]. Tested on
|
||||
// "local" and "drive" backends.
|
||||
// 2. Add support for the ASYNC protocol extension. This may improve performance.
|
||||
// 3. Support the [simple export interface]. This will enable `git-annex
|
||||
// export` functionality.
|
||||
// 4. Once the draft is finalized, support import/export interface.
|
||||
//
|
||||
// [git-annex]: https://git-annex.branchable.com/
|
||||
// [external special remote protocol]: https://git-annex.branchable.com/design/external_special_remote_protocol/
|
||||
// [simple export interface]: https://git-annex.branchable.com/design/external_special_remote_protocol/export_and_import_appendix/
|
||||
// [issue #7625]: https://github.com/rclone/rclone/issues/7625
|
||||
package gitannex
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
_ "embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
const subcommandName string = "gitannex"
|
||||
const uniqueCommandName string = "git-annex-remote-rclone-builtin"
|
||||
|
||||
//go:embed gitannex.md
|
||||
var gitannexHelp string
|
||||
|
||||
func init() {
|
||||
os.Args = maybeTransformArgs(os.Args)
|
||||
cmd.Root.AddCommand(command)
|
||||
}
|
||||
|
||||
// maybeTransformArgs returns a modified version of `args` with the "gitannex"
|
||||
// subcommand inserted when `args` indicates that the program was executed as
|
||||
// "git-annex-remote-rclone-builtin". One way this can happen is when rclone is
|
||||
// invoked via symlink. Otherwise, returns `args`.
|
||||
func maybeTransformArgs(args []string) []string {
|
||||
if len(args) == 0 || filepath.Base(args[0]) != uniqueCommandName {
|
||||
return args
|
||||
}
|
||||
newArgs := make([]string, 0, len(args)+1)
|
||||
newArgs = append(newArgs, args[0])
|
||||
newArgs = append(newArgs, subcommandName)
|
||||
newArgs = append(newArgs, args[1:]...)
|
||||
return newArgs
|
||||
}
|
||||
|
||||
// messageParser helps parse messages we receive from git-annex into a sequence
|
||||
// of parameters. Messages are not quite trivial to parse because they are
|
||||
// separated by spaces, but the final parameter may itself contain spaces.
|
||||
//
|
||||
// This abstraction is necessary because simply splitting on space doesn't cut
|
||||
// it. Also, we cannot know how many parameters to parse until we've parsed the
|
||||
// first parameter.
|
||||
type messageParser struct {
|
||||
line string
|
||||
}
|
||||
|
||||
// nextSpaceDelimitedParameter consumes the next space-delimited parameter.
|
||||
func (m *messageParser) nextSpaceDelimitedParameter() (string, error) {
|
||||
m.line = strings.TrimRight(m.line, "\r\n")
|
||||
if len(m.line) == 0 {
|
||||
return "", errors.New("nothing remains to parse")
|
||||
}
|
||||
|
||||
before, after, found := strings.Cut(m.line, " ")
|
||||
if found {
|
||||
if len(before) == 0 {
|
||||
return "", fmt.Errorf("found an empty space-delimited parameter in line: %q", m.line)
|
||||
}
|
||||
m.line = after
|
||||
return before, nil
|
||||
}
|
||||
|
||||
remaining := m.line
|
||||
m.line = ""
|
||||
return remaining, nil
|
||||
}
|
||||
|
||||
// finalParameter consumes the final parameter, which may contain spaces.
|
||||
func (m *messageParser) finalParameter() (string, error) {
|
||||
m.line = strings.TrimRight(m.line, "\r\n")
|
||||
if len(m.line) == 0 {
|
||||
return "", errors.New("nothing remains to parse")
|
||||
}
|
||||
|
||||
param := m.line
|
||||
m.line = ""
|
||||
return param, nil
|
||||
}
|
||||
|
||||
// configDefinition describes a configuration value required by this command. We
|
||||
// use "GETCONFIG" messages to query git-annex for these values at runtime.
|
||||
type configDefinition struct {
|
||||
name string
|
||||
description string
|
||||
destination *string
|
||||
}
|
||||
|
||||
// server contains this command's current state.
|
||||
type server struct {
|
||||
reader *bufio.Reader
|
||||
writer io.Writer
|
||||
|
||||
// When true, the server prints a transcript of messages sent and received
|
||||
// to stderr.
|
||||
verbose bool
|
||||
|
||||
extensionInfo bool
|
||||
extensionAsync bool
|
||||
extensionGetGitRemoteName bool
|
||||
extensionUnavailableResponse bool
|
||||
|
||||
configsDone bool
|
||||
configPrefix string
|
||||
configRcloneRemoteName string
|
||||
}
|
||||
|
||||
func (s *server) sendMsg(msg string) {
|
||||
msg = msg + "\n"
|
||||
if _, err := io.WriteString(s.writer, msg); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if s.verbose {
|
||||
_, err := os.Stderr.WriteString(fmt.Sprintf("server sent %q\n", msg))
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to write verbose message to stderr: %w", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *server) getMsg() (*messageParser, error) {
|
||||
msg, err := s.reader.ReadString('\n')
|
||||
if err != nil {
|
||||
if len(msg) == 0 {
|
||||
// Git-annex closes stdin when it is done with us, so failing to
|
||||
// read a new line is not an error.
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("expected message to end with newline: %q", msg)
|
||||
}
|
||||
if s.verbose {
|
||||
_, err := os.Stderr.WriteString(fmt.Sprintf("server received %q\n", msg))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to write verbose message to stderr: %w", err)
|
||||
}
|
||||
}
|
||||
return &messageParser{msg}, nil
|
||||
}
|
||||
|
||||
func (s *server) run() error {
|
||||
// The remote sends the first message.
|
||||
s.sendMsg("VERSION 2")
|
||||
|
||||
for {
|
||||
message, err := s.getMsg()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error receiving message: %w", err)
|
||||
}
|
||||
|
||||
if message == nil {
|
||||
break
|
||||
}
|
||||
|
||||
command, err := message.nextSpaceDelimitedParameter()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse command")
|
||||
}
|
||||
|
||||
switch command {
|
||||
//
|
||||
// Git-annex requires that these requests are supported.
|
||||
//
|
||||
case "INITREMOTE":
|
||||
err = s.handleInitRemote()
|
||||
case "PREPARE":
|
||||
err = s.handlePrepare()
|
||||
case "EXPORTSUPPORTED":
|
||||
// Indicate that we do not support exports.
|
||||
s.sendMsg("EXPORTSUPPORTED-FAILURE")
|
||||
case "TRANSFER":
|
||||
err = s.handleTransfer(message)
|
||||
case "CHECKPRESENT":
|
||||
err = s.handleCheckPresent(message)
|
||||
case "REMOVE":
|
||||
err = s.handleRemove(message)
|
||||
case "ERROR":
|
||||
errorMessage, parseErr := message.finalParameter()
|
||||
if parseErr != nil {
|
||||
err = fmt.Errorf("error while parsing ERROR message from git-annex: %w", parseErr)
|
||||
break
|
||||
}
|
||||
err = fmt.Errorf("received error message from git-annex: %s", errorMessage)
|
||||
|
||||
//
|
||||
// These requests are optional.
|
||||
//
|
||||
case "EXTENSIONS":
|
||||
// Git-annex just told us which protocol extensions it supports.
|
||||
// Respond with the list of extensions that we want to use (none).
|
||||
err = s.handleExtensions(message)
|
||||
case "LISTCONFIGS":
|
||||
s.handleListConfigs()
|
||||
case "GETCOST":
|
||||
// Git-annex wants to know the "cost" of using this remote. It
|
||||
// probably depends on the backend we will be using, but let's just
|
||||
// consider this an "expensive remote" per git-annex's
|
||||
// Config/Cost.hs.
|
||||
s.sendMsg("COST 200")
|
||||
case "GETAVAILABILITY":
|
||||
// Indicate that this is a cloud service.
|
||||
s.sendMsg("AVAILABILITY GLOBAL")
|
||||
case "CLAIMURL", "CHECKURL", "WHEREIS", "GETINFO":
|
||||
s.sendMsg("UNSUPPORTED-REQUEST")
|
||||
default:
|
||||
err = fmt.Errorf("received unexpected message from git-annex: %s", message.line)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Idempotently handle an incoming INITREMOTE message. This should perform
|
||||
// one-time setup operations, but we may receive the command again, e.g. when
|
||||
// this git-annex remote is initialized in a different repository.
|
||||
func (s *server) handleInitRemote() error {
|
||||
if err := s.queryConfigs(); err != nil {
|
||||
return fmt.Errorf("failed to get configs: %w", err)
|
||||
}
|
||||
|
||||
remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName))
|
||||
if err != nil {
|
||||
s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote")
|
||||
return fmt.Errorf("failed to open root directory of rclone remote: %w", err)
|
||||
}
|
||||
|
||||
if !remoteRootFs.Features().CanHaveEmptyDirectories {
|
||||
s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories")
|
||||
return fmt.Errorf("rclone remote does not support empty directories")
|
||||
}
|
||||
|
||||
if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil {
|
||||
s.sendMsg("INITREMOTE-FAILURE failed to mkdir")
|
||||
return fmt.Errorf("failed to mkdir: %w", err)
|
||||
}
|
||||
|
||||
s.sendMsg("INITREMOTE-SUCCESS")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get a list of configs with pointers to fields of `s`.
|
||||
func (s *server) getRequiredConfigs() []configDefinition {
|
||||
return []configDefinition{
|
||||
{
|
||||
"rcloneremotename",
|
||||
"Name of the rclone remote to use. " +
|
||||
"Must match a remote known to rclone. " +
|
||||
"(Note that rclone remotes are a distinct concept from git-annex remotes.)",
|
||||
&s.configRcloneRemoteName,
|
||||
},
|
||||
{
|
||||
"rcloneprefix",
|
||||
"Directory where rclone will write git-annex content. " +
|
||||
"If not specified, defaults to \"git-annex-rclone\". " +
|
||||
"This directory be created on init if it does not exist.",
|
||||
&s.configPrefix,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Query git-annex for config values.
|
||||
func (s *server) queryConfigs() error {
|
||||
if s.configsDone {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Send a "GETCONFIG" message for each required config and parse git-annex's
|
||||
// "VALUE" response.
|
||||
for _, config := range s.getRequiredConfigs() {
|
||||
s.sendMsg(fmt.Sprintf("GETCONFIG %s", config.name))
|
||||
|
||||
message, err := s.getMsg()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
valueKeyword, err := message.nextSpaceDelimitedParameter()
|
||||
if err != nil || valueKeyword != "VALUE" {
|
||||
return fmt.Errorf("failed to parse config value: %s %s", valueKeyword, message.line)
|
||||
}
|
||||
|
||||
value, err := message.finalParameter()
|
||||
if err != nil || value == "" {
|
||||
return fmt.Errorf("config value of %q must not be empty", config.name)
|
||||
}
|
||||
|
||||
*config.destination = value
|
||||
}
|
||||
|
||||
s.configsDone = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *server) handlePrepare() error {
|
||||
if err := s.queryConfigs(); err != nil {
|
||||
s.sendMsg("PREPARE-FAILURE Error getting configs")
|
||||
return fmt.Errorf("error getting configs: %w", err)
|
||||
}
|
||||
s.sendMsg("PREPARE-SUCCESS")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Git-annex is asking us to return the list of settings that we use. Keep this
|
||||
// in sync with `handlePrepare()`.
|
||||
func (s *server) handleListConfigs() {
|
||||
for _, config := range s.getRequiredConfigs() {
|
||||
s.sendMsg(fmt.Sprintf("CONFIG %s %s", config.name, config.description))
|
||||
}
|
||||
s.sendMsg("CONFIGEND")
|
||||
}
|
||||
|
||||
func (s *server) handleTransfer(message *messageParser) error {
|
||||
argMode, err := message.nextSpaceDelimitedParameter()
|
||||
if err != nil {
|
||||
s.sendMsg("TRANSFER-FAILURE failed to parse direction")
|
||||
return fmt.Errorf("malformed arguments for TRANSFER: %w", err)
|
||||
}
|
||||
argKey, err := message.nextSpaceDelimitedParameter()
|
||||
if err != nil {
|
||||
s.sendMsg("TRANSFER-FAILURE failed to parse key")
|
||||
return fmt.Errorf("malformed arguments for TRANSFER: %w", err)
|
||||
}
|
||||
argFile, err := message.finalParameter()
|
||||
if err != nil {
|
||||
s.sendMsg("TRANSFER-FAILURE failed to parse file")
|
||||
return fmt.Errorf("malformed arguments for TRANSFER: %w", err)
|
||||
}
|
||||
|
||||
if err := s.queryConfigs(); err != nil {
|
||||
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get configs", argMode, argKey))
|
||||
return fmt.Errorf("error getting configs: %w", err)
|
||||
}
|
||||
|
||||
remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey))
|
||||
return err
|
||||
}
|
||||
|
||||
localDir := filepath.Dir(argFile)
|
||||
localFs, err := cache.Get(context.TODO(), localDir)
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get local fs", argMode, argKey))
|
||||
return fmt.Errorf("failed to get local fs: %w", err)
|
||||
}
|
||||
|
||||
remoteFileName := argKey
|
||||
localFileName := filepath.Base(argFile)
|
||||
|
||||
switch argMode {
|
||||
case "STORE":
|
||||
err = operations.CopyFile(context.TODO(), remoteFs, localFs, remoteFileName, localFileName)
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err))
|
||||
return err
|
||||
}
|
||||
|
||||
case "RETRIEVE":
|
||||
err = operations.CopyFile(context.TODO(), localFs, remoteFs, localFileName, remoteFileName)
|
||||
// It is non-fatal when retrieval fails because the file is missing on
|
||||
// the remote.
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s not found", argMode, argKey))
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err))
|
||||
return err
|
||||
}
|
||||
|
||||
default:
|
||||
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s unrecognized mode", argMode, argKey))
|
||||
return fmt.Errorf("received malformed TRANSFER mode: %v", argMode)
|
||||
}
|
||||
|
||||
s.sendMsg(fmt.Sprintf("TRANSFER-SUCCESS %s %s", argMode, argKey))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *server) handleCheckPresent(message *messageParser) error {
|
||||
argKey, err := message.finalParameter()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.queryConfigs(); err != nil {
|
||||
s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s failed to get configs", argKey))
|
||||
return fmt.Errorf("error getting configs: %s", err)
|
||||
}
|
||||
|
||||
remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey))
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = remoteFs.NewObject(context.TODO(), argKey)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s error finding file", argKey))
|
||||
return err
|
||||
}
|
||||
|
||||
s.sendMsg(fmt.Sprintf("CHECKPRESENT-SUCCESS %s", argKey))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *server) handleRemove(message *messageParser) error {
|
||||
argKey, err := message.finalParameter()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
|
||||
return fmt.Errorf("error getting remote fs: %w", err)
|
||||
}
|
||||
|
||||
fileObj, err := remoteFs.NewObject(context.TODO(), argKey)
|
||||
// It is non-fatal when removal fails because the file is missing on the
|
||||
// remote.
|
||||
if errors.Is(err, fs.ErrorObjectNotFound) {
|
||||
s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey))
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error getting new fs object: %s", argKey, err))
|
||||
return fmt.Errorf("error getting new fs object: %w", err)
|
||||
}
|
||||
if err := operations.DeleteFile(context.TODO(), fileObj); err != nil {
|
||||
s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error deleting file", argKey))
|
||||
return fmt.Errorf("error deleting file: %q", argKey)
|
||||
}
|
||||
s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *server) handleExtensions(message *messageParser) error {
|
||||
for {
|
||||
extension, err := message.nextSpaceDelimitedParameter()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
switch extension {
|
||||
case "INFO":
|
||||
s.extensionInfo = true
|
||||
case "ASYNC":
|
||||
s.extensionAsync = true
|
||||
case "GETGITREMOTENAME":
|
||||
s.extensionGetGitRemoteName = true
|
||||
case "UNAVAILABLERESPONSE":
|
||||
s.extensionUnavailableResponse = true
|
||||
}
|
||||
}
|
||||
s.sendMsg("EXTENSIONS")
|
||||
return nil
|
||||
}
|
||||
|
||||
var command = &cobra.Command{
|
||||
Aliases: []string{uniqueCommandName},
|
||||
Use: subcommandName,
|
||||
Short: "Speaks with git-annex over stdin/stdout.",
|
||||
Long: gitannexHelp,
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.67.0",
|
||||
},
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(0, 0, command, args)
|
||||
|
||||
s := server{
|
||||
reader: bufio.NewReader(os.Stdin),
|
||||
writer: os.Stdout,
|
||||
}
|
||||
err := s.run()
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("ERROR %s", err.Error()))
|
||||
panic(err)
|
||||
}
|
||||
},
|
||||
}
|
||||
38
cmd/gitannex/gitannex.md
Normal file
38
cmd/gitannex/gitannex.md
Normal file
@ -0,0 +1,38 @@
|
||||
Rclone's gitannex subcommand enables git-annex to store and retrieve content
|
||||
from an rclone remote. It expects to be run by git-annex, not directly by users.
|
||||
It is an "external special remote program" as defined by git-annex.
|
||||
|
||||
Installation on Linux
|
||||
---------------------
|
||||
|
||||
1. Create a symlink and ensure it's on your PATH. For example:
|
||||
|
||||
ln -s "$(realpath rclone)" "$HOME/bin/git-annex-remote-rclone-builtin"
|
||||
|
||||
2. Add a new external remote to your git-annex repo.
|
||||
|
||||
The new remote's type should be "rclone-builtin". When git-annex interacts
|
||||
with remotes of this type, it will try to run a command named
|
||||
"git-annex-remote-rclone-builtin", so the symlink from the previous step
|
||||
should be on your PATH.
|
||||
|
||||
The following example creates a new git-annex remote named "MyRemote" that
|
||||
will use the rclone remote named "SomeRcloneRemote". This rclone remote must
|
||||
be configured in your rclone.conf file, wherever that is located on your
|
||||
system. The rcloneprefix value ensures that content is only written into the
|
||||
rclone remote underneath the "git-annex-content" directory.
|
||||
|
||||
git annex initremote MyRemote \
|
||||
type=external \
|
||||
externaltype=rclone-builtin \
|
||||
encryption=none \
|
||||
rcloneremotename=SomeRcloneRemote \
|
||||
rcloneprefix=git-annex-content
|
||||
|
||||
3. Before you trust this command with your precious data, be sure to **test the
|
||||
remote**. This command is very new and has not been tested on many rclone
|
||||
backends. Caveat emptor!
|
||||
|
||||
git annex testremote my-rclone-remote
|
||||
|
||||
Happy annexing!
|
||||
969
cmd/gitannex/gitannex_test.go
Normal file
969
cmd/gitannex/gitannex_test.go
Normal file
@ -0,0 +1,969 @@
|
||||
package gitannex
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
// Without this import, the local filesystem backend would be unavailable.
|
||||
// It looks unused, but the act of importing it runs its `init()` function.
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/rclone/rclone/fstest/mockfs"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFixArgsForSymlinkIdentity(t *testing.T) {
|
||||
for _, argList := range [][]string{
|
||||
[]string{},
|
||||
[]string{"foo"},
|
||||
[]string{"foo", "bar"},
|
||||
[]string{"foo", "bar", "baz"},
|
||||
} {
|
||||
assert.Equal(t, maybeTransformArgs(argList), argList)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFixArgsForSymlinkCorrectName(t *testing.T) {
|
||||
assert.Equal(t,
|
||||
maybeTransformArgs([]string{"git-annex-remote-rclone-builtin"}),
|
||||
[]string{"git-annex-remote-rclone-builtin", "gitannex"})
|
||||
assert.Equal(t,
|
||||
maybeTransformArgs([]string{"/path/to/git-annex-remote-rclone-builtin"}),
|
||||
[]string{"/path/to/git-annex-remote-rclone-builtin", "gitannex"})
|
||||
}
|
||||
|
||||
type messageParserTestCase struct {
|
||||
label string
|
||||
testFunc func(*testing.T)
|
||||
}
|
||||
|
||||
var messageParserTestCases = []messageParserTestCase{
|
||||
{
|
||||
"OneParam",
|
||||
func(t *testing.T) {
|
||||
m := messageParser{"foo\n"}
|
||||
|
||||
param, err := m.nextSpaceDelimitedParameter()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, param, "foo")
|
||||
|
||||
param, err = m.nextSpaceDelimitedParameter()
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, param, "")
|
||||
|
||||
param, err = m.finalParameter()
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, param, "")
|
||||
|
||||
param, err = m.finalParameter()
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, param, "")
|
||||
|
||||
param, err = m.nextSpaceDelimitedParameter()
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, param, "")
|
||||
|
||||
},
|
||||
},
|
||||
{
|
||||
"TwoParams",
|
||||
func(t *testing.T) {
|
||||
m := messageParser{"foo bar\n"}
|
||||
|
||||
param, err := m.nextSpaceDelimitedParameter()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, param, "foo")
|
||||
|
||||
param, err = m.nextSpaceDelimitedParameter()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, param, "bar")
|
||||
|
||||
param, err = m.nextSpaceDelimitedParameter()
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, param, "")
|
||||
|
||||
param, err = m.finalParameter()
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, param, "")
|
||||
},
|
||||
},
|
||||
{
|
||||
"TwoParamsNoTrailingNewline",
|
||||
|
||||
func(t *testing.T) {
|
||||
m := messageParser{"foo bar"}
|
||||
|
||||
param, err := m.nextSpaceDelimitedParameter()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, param, "foo")
|
||||
|
||||
param, err = m.nextSpaceDelimitedParameter()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, param, "bar")
|
||||
|
||||
param, err = m.nextSpaceDelimitedParameter()
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, param, "")
|
||||
|
||||
param, err = m.finalParameter()
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, param, "")
|
||||
},
|
||||
},
|
||||
{
|
||||
"ThreeParamsWhereFinalParamContainsSpaces",
|
||||
func(t *testing.T) {
|
||||
m := messageParser{"firstparam secondparam final param with spaces"}
|
||||
|
||||
param, err := m.nextSpaceDelimitedParameter()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, param, "firstparam")
|
||||
|
||||
param, err = m.nextSpaceDelimitedParameter()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, param, "secondparam")
|
||||
|
||||
param, err = m.finalParameter()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, param, "final param with spaces")
|
||||
},
|
||||
},
|
||||
{
|
||||
"OneLongFinalParameter",
|
||||
func(t *testing.T) {
|
||||
for _, lineEnding := range []string{"", "\n", "\r", "\r\n", "\n\r"} {
|
||||
lineEnding := lineEnding
|
||||
testName := fmt.Sprintf("lineEnding%x", lineEnding)
|
||||
|
||||
t.Run(testName, func(t *testing.T) {
|
||||
m := messageParser{"one long final parameter" + lineEnding}
|
||||
|
||||
param, err := m.finalParameter()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, param, "one long final parameter")
|
||||
|
||||
param, err = m.finalParameter()
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, param, "")
|
||||
})
|
||||
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"MultipleSpaces",
|
||||
func(t *testing.T) {
|
||||
m := messageParser{"foo bar\n\r"}
|
||||
|
||||
param, err := m.nextSpaceDelimitedParameter()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, param, "foo")
|
||||
|
||||
param, err = m.nextSpaceDelimitedParameter()
|
||||
assert.Error(t, err, "blah")
|
||||
assert.Equal(t, param, "")
|
||||
},
|
||||
},
|
||||
{
|
||||
"StartsWithSpace",
|
||||
func(t *testing.T) {
|
||||
m := messageParser{" foo"}
|
||||
|
||||
param, err := m.nextSpaceDelimitedParameter()
|
||||
assert.Error(t, err, "blah")
|
||||
assert.Equal(t, param, "")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestMessageParser(t *testing.T) {
|
||||
for _, testCase := range messageParserTestCases {
|
||||
testCase := testCase
|
||||
t.Run(testCase.label, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
testCase.testFunc(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type testState struct {
|
||||
t *testing.T
|
||||
server *server
|
||||
mockStdinW *io.PipeWriter
|
||||
mockStdoutReader *bufio.Reader
|
||||
|
||||
localFsDir string
|
||||
configPath string
|
||||
remoteName string
|
||||
}
|
||||
|
||||
func makeTestState(t *testing.T) testState {
|
||||
stdinR, stdinW := io.Pipe()
|
||||
stdoutR, stdoutW := io.Pipe()
|
||||
|
||||
return testState{
|
||||
t: t,
|
||||
server: &server{
|
||||
reader: bufio.NewReader(stdinR),
|
||||
writer: stdoutW,
|
||||
},
|
||||
mockStdinW: stdinW,
|
||||
mockStdoutReader: bufio.NewReader(stdoutR),
|
||||
}
|
||||
}
|
||||
|
||||
func (h *testState) requireReadLineExact(line string) {
|
||||
receivedLine, err := h.mockStdoutReader.ReadString('\n')
|
||||
require.NoError(h.t, err)
|
||||
require.Equal(h.t, line+"\n", receivedLine)
|
||||
}
|
||||
|
||||
func (h *testState) requireWriteLine(line string) {
|
||||
_, err := h.mockStdinW.Write([]byte(line + "\n"))
|
||||
require.NoError(h.t, err)
|
||||
}
|
||||
|
||||
// Preconfigure the handle. This enables the calling test to skip the PREPARE
|
||||
// handshake.
|
||||
func (h *testState) preconfigureServer() {
|
||||
h.server.configPrefix = h.localFsDir
|
||||
h.server.configRcloneRemoteName = h.remoteName
|
||||
h.server.configsDone = true
|
||||
}
|
||||
|
||||
// getUniqueRemoteName returns a valid remote name derived from the given test's
|
||||
// name. This is necessary because when a test registers a second remote with
|
||||
// the same name, the original remote appears to take precedence. This function
|
||||
// is injective, so each test gets a unique remote name. Returned strings
|
||||
// contain no spaces.
|
||||
func getUniqueRemoteName(t *testing.T) string {
|
||||
// Using sha256 as a hack to ensure injectivity without adding a global
|
||||
// variable.
|
||||
return fmt.Sprintf("remote-%x", sha256.Sum256([]byte(t.Name())))
|
||||
}
|
||||
|
||||
type testCase struct {
|
||||
label string
|
||||
testProtocolFunc func(*testing.T, *testState)
|
||||
expectedError string
|
||||
}
|
||||
|
||||
// These test cases run against the "local" backend.
|
||||
var localBackendTestCases = []testCase{
|
||||
{
|
||||
label: "HandlesInit",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "HandlesPrepare",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
|
||||
if !h.server.extensionInfo {
|
||||
t.Errorf("expected INFO extension to be enabled")
|
||||
return
|
||||
}
|
||||
|
||||
h.requireWriteLine("PREPARE")
|
||||
h.requireReadLineExact("GETCONFIG rcloneremotename")
|
||||
h.requireWriteLine("VALUE " + h.remoteName)
|
||||
h.requireReadLineExact("GETCONFIG rcloneprefix")
|
||||
h.requireWriteLine("VALUE " + h.localFsDir)
|
||||
h.requireReadLineExact("PREPARE-SUCCESS")
|
||||
|
||||
require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
|
||||
require.Equal(t, h.server.configPrefix, h.localFsDir)
|
||||
require.True(t, h.server.configsDone)
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "HandlesPrepareAndDoesNotTrimWhitespaceFromValue",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
|
||||
if !h.server.extensionInfo {
|
||||
t.Errorf("expected INFO extension to be enabled")
|
||||
return
|
||||
}
|
||||
|
||||
h.requireWriteLine("PREPARE")
|
||||
h.requireReadLineExact("GETCONFIG rcloneremotename")
|
||||
|
||||
remoteNameWithSpaces := fmt.Sprintf(" %s ", h.remoteName)
|
||||
localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)
|
||||
|
||||
h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))
|
||||
h.requireReadLineExact("GETCONFIG rcloneprefix")
|
||||
|
||||
h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))
|
||||
h.requireReadLineExact("PREPARE-SUCCESS")
|
||||
|
||||
require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)
|
||||
require.Equal(t, h.server.configPrefix, localFsDirWithSpaces)
|
||||
require.True(t, h.server.configsDone)
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "HandlesEarlyError",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("ERROR foo")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
expectedError: "received error message from git-annex: foo",
|
||||
},
|
||||
// Test what happens when the git-annex client sends "GETCONFIG", but
|
||||
// doesn't understand git-annex's response.
|
||||
{
|
||||
label: "ConfigFail",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.True(t, h.server.extensionInfo)
|
||||
|
||||
h.requireWriteLine("PREPARE")
|
||||
h.requireReadLineExact("GETCONFIG rcloneremotename")
|
||||
h.requireWriteLine("ERROR ineffable error")
|
||||
h.requireReadLineExact("PREPARE-FAILURE Error getting configs")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
expectedError: "failed to parse config value: ERROR ineffable error",
|
||||
},
|
||||
{
|
||||
label: "TransferStoreEmptyPath",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
// Note the whitespace following the key.
|
||||
h.requireWriteLine("TRANSFER STORE Key ")
|
||||
h.requireReadLineExact("TRANSFER-FAILURE failed to parse file")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
expectedError: "malformed arguments for TRANSFER: nothing remains to parse",
|
||||
},
|
||||
// Repeated EXTENSIONS messages add to each other rather than overriding
|
||||
// prior advertised extensions. This behavior is not mandated by the
|
||||
// protocol design.
|
||||
{
|
||||
label: "ExtensionsCompound",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
h.requireWriteLine("EXTENSIONS")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.False(t, h.server.extensionInfo)
|
||||
require.False(t, h.server.extensionAsync)
|
||||
require.False(t, h.server.extensionGetGitRemoteName)
|
||||
require.False(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
h.requireWriteLine("EXTENSIONS INFO")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.True(t, h.server.extensionInfo)
|
||||
require.False(t, h.server.extensionAsync)
|
||||
require.False(t, h.server.extensionGetGitRemoteName)
|
||||
require.False(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
h.requireWriteLine("EXTENSIONS ASYNC")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.True(t, h.server.extensionInfo)
|
||||
require.True(t, h.server.extensionAsync)
|
||||
require.False(t, h.server.extensionGetGitRemoteName)
|
||||
require.False(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
h.requireWriteLine("EXTENSIONS GETGITREMOTENAME")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.True(t, h.server.extensionInfo)
|
||||
require.True(t, h.server.extensionAsync)
|
||||
require.True(t, h.server.extensionGetGitRemoteName)
|
||||
require.False(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
h.requireWriteLine("EXTENSIONS UNAVAILABLERESPONSE")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.True(t, h.server.extensionInfo)
|
||||
require.True(t, h.server.extensionAsync)
|
||||
require.True(t, h.server.extensionGetGitRemoteName)
|
||||
require.True(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "ExtensionsIdempotent",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
h.requireWriteLine("EXTENSIONS")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.False(t, h.server.extensionInfo)
|
||||
require.False(t, h.server.extensionAsync)
|
||||
require.False(t, h.server.extensionGetGitRemoteName)
|
||||
require.False(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
h.requireWriteLine("EXTENSIONS")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.False(t, h.server.extensionInfo)
|
||||
require.False(t, h.server.extensionAsync)
|
||||
require.False(t, h.server.extensionGetGitRemoteName)
|
||||
require.False(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
h.requireWriteLine("EXTENSIONS INFO")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.True(t, h.server.extensionInfo)
|
||||
require.False(t, h.server.extensionAsync)
|
||||
require.False(t, h.server.extensionGetGitRemoteName)
|
||||
require.False(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
h.requireWriteLine("EXTENSIONS INFO")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.True(t, h.server.extensionInfo)
|
||||
require.False(t, h.server.extensionAsync)
|
||||
require.False(t, h.server.extensionGetGitRemoteName)
|
||||
require.False(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
h.requireWriteLine("EXTENSIONS ASYNC ASYNC")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.True(t, h.server.extensionInfo)
|
||||
require.True(t, h.server.extensionAsync)
|
||||
require.False(t, h.server.extensionGetGitRemoteName)
|
||||
require.False(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "ExtensionsSupportsMultiple",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
h.requireWriteLine("EXTENSIONS")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.False(t, h.server.extensionInfo)
|
||||
require.False(t, h.server.extensionAsync)
|
||||
require.False(t, h.server.extensionGetGitRemoteName)
|
||||
require.False(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
h.requireWriteLine("EXTENSIONS INFO ASYNC")
|
||||
h.requireReadLineExact("EXTENSIONS")
|
||||
require.True(t, h.server.extensionInfo)
|
||||
require.True(t, h.server.extensionAsync)
|
||||
require.False(t, h.server.extensionGetGitRemoteName)
|
||||
require.False(t, h.server.extensionUnavailableResponse)
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "TransferStoreAbsolute",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
// Create temp file for transfer with an absolute path.
|
||||
fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
|
||||
require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
|
||||
require.FileExists(t, fileToTransfer)
|
||||
require.True(t, filepath.IsAbs(fileToTransfer))
|
||||
|
||||
// Specify an absolute path to transfer.
|
||||
h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
|
||||
h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
|
||||
require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
|
||||
|
||||
// Transfer the same absolute path a second time, but with a different key.
|
||||
h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + fileToTransfer)
|
||||
h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute2")
|
||||
require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute2"))
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT KeyAbsolute2")
|
||||
h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyAbsolute2")
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
|
||||
h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
// Test that the TRANSFER command understands simple relative paths
|
||||
// consisting only of a file name.
|
||||
{
|
||||
label: "TransferStoreRelative",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
// Save the current working directory so we can restore it when this
|
||||
// test ends.
|
||||
cwd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, os.Chdir(t.TempDir()))
|
||||
t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
// Create temp file for transfer with a relative path.
|
||||
fileToTransfer := "file.txt"
|
||||
require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
|
||||
require.FileExists(t, fileToTransfer)
|
||||
require.False(t, filepath.IsAbs(fileToTransfer))
|
||||
|
||||
// Specify a relative path to transfer.
|
||||
h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
|
||||
h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
|
||||
require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT KeyRelative")
|
||||
h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
|
||||
h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "TransferStorePathWithInteriorWhitespace",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
// Save the current working directory so we can restore it when this
|
||||
// test ends.
|
||||
cwd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, os.Chdir(t.TempDir()))
|
||||
t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
|
||||
|
||||
h.preconfigureServer()
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
// Create temp file for transfer.
|
||||
fileToTransfer := "filename with spaces.txt"
|
||||
require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
|
||||
require.FileExists(t, fileToTransfer)
|
||||
require.False(t, filepath.IsAbs(fileToTransfer))
|
||||
|
||||
// Specify a relative path to transfer.
|
||||
h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
|
||||
h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
|
||||
require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT KeyRelative")
|
||||
h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
|
||||
h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "CheckPresentAndTransfer",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
// Create temp file for transfer.
|
||||
fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
|
||||
require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
|
||||
h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
|
||||
|
||||
// Specify an absolute path to transfer.
|
||||
require.True(t, filepath.IsAbs(fileToTransfer))
|
||||
h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
|
||||
h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
|
||||
require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
// Check whether a key is present, transfer a file with that key, then check
|
||||
// again whether it is present.
|
||||
//
|
||||
// This is a regression test for a bug where the second CHECKPRESENT would
|
||||
// generate the following response:
|
||||
//
|
||||
// CHECKPRESENT-UNKNOWN ${key} failed to read directory entry: readdirent ${filepath}: not a directory
|
||||
//
|
||||
// This message was generated by the local backend's `List()` function. When
|
||||
// checking whether a file exists, we were erroneously listing its contents as
|
||||
// if it were a directory.
|
||||
{
|
||||
label: "CheckpresentTransferCheckpresent",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
// Create temp file for transfer.
|
||||
fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
|
||||
require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT foo")
|
||||
h.requireReadLineExact("CHECKPRESENT-FAILURE foo")
|
||||
|
||||
h.requireWriteLine("TRANSFER STORE foo " + fileToTransfer)
|
||||
h.requireReadLineExact("TRANSFER-SUCCESS STORE foo")
|
||||
require.FileExists(t, filepath.Join(h.localFsDir, "foo"))
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT foo")
|
||||
h.requireReadLineExact("CHECKPRESENT-SUCCESS foo")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "TransferAndCheckpresentWithRealisticKey",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
// Create temp file for transfer.
|
||||
fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
|
||||
require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
realisticKey := "SHA256E-s1048576--7ba87e06b9b7903cfbaf4a38736766c161e3e7b42f06fe57f040aa410a8f0701.this-is-a-test-key"
|
||||
|
||||
// Specify an absolute path to transfer.
|
||||
require.True(t, filepath.IsAbs(fileToTransfer))
|
||||
h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, fileToTransfer))
|
||||
h.requireReadLineExact("TRANSFER-SUCCESS STORE " + realisticKey)
|
||||
require.FileExists(t, filepath.Join(h.localFsDir, realisticKey))
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT " + realisticKey)
|
||||
h.requireReadLineExact("CHECKPRESENT-SUCCESS " + realisticKey)
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "RetrieveNonexistentFile",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
h.requireWriteLine("TRANSFER RETRIEVE SomeKey path")
|
||||
h.requireReadLineExact("TRANSFER-FAILURE RETRIEVE SomeKey not found")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "StoreCheckpresentRetrieve",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
// Create temp file for transfer.
|
||||
fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
|
||||
require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
// Specify an absolute path to transfer.
|
||||
require.True(t, filepath.IsAbs(fileToTransfer))
|
||||
h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
|
||||
h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
|
||||
require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT SomeKey")
|
||||
h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
|
||||
|
||||
retrievedFilePath := fileToTransfer + ".retrieved"
|
||||
require.NoFileExists(t, retrievedFilePath)
|
||||
h.requireWriteLine("TRANSFER RETRIEVE SomeKey " + retrievedFilePath)
|
||||
h.requireReadLineExact("TRANSFER-SUCCESS RETRIEVE SomeKey")
|
||||
require.FileExists(t, retrievedFilePath)
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "RemovePreexistingFile",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
// Write a file into the remote without using the git-annex
|
||||
// protocol.
|
||||
remoteFilePath := filepath.Join(h.localFsDir, "SomeKey")
|
||||
require.NoError(t, os.WriteFile(remoteFilePath, []byte("HELLO"), 0600))
|
||||
require.FileExists(t, remoteFilePath)
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT SomeKey")
|
||||
h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
|
||||
require.FileExists(t, remoteFilePath)
|
||||
|
||||
h.requireWriteLine("REMOVE SomeKey")
|
||||
h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
|
||||
require.NoFileExists(t, remoteFilePath)
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT SomeKey")
|
||||
h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
|
||||
require.NoFileExists(t, remoteFilePath)
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Remove",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
// Create temp file for transfer.
|
||||
fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
|
||||
require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT SomeKey")
|
||||
h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
|
||||
|
||||
// Specify an absolute path to transfer.
|
||||
require.True(t, filepath.IsAbs(fileToTransfer))
|
||||
h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
|
||||
h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
|
||||
require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT SomeKey")
|
||||
h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
|
||||
|
||||
h.requireWriteLine("REMOVE SomeKey")
|
||||
h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
|
||||
require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT SomeKey")
|
||||
h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "RemoveNonexistentFile",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
// Create temp file for transfer.
|
||||
fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
|
||||
require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT SomeKey")
|
||||
h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
|
||||
|
||||
require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
|
||||
h.requireWriteLine("REMOVE SomeKey")
|
||||
h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
|
||||
require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
|
||||
|
||||
h.requireWriteLine("CHECKPRESENT SomeKey")
|
||||
h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "ExportNotSupported",
|
||||
testProtocolFunc: func(t *testing.T, h *testState) {
|
||||
h.preconfigureServer()
|
||||
|
||||
h.requireReadLineExact("VERSION 2")
|
||||
h.requireWriteLine("INITREMOTE")
|
||||
h.requireReadLineExact("INITREMOTE-SUCCESS")
|
||||
|
||||
h.requireWriteLine("EXPORTSUPPORTED")
|
||||
h.requireReadLineExact("EXPORTSUPPORTED-FAILURE")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestGitAnnexLocalBackendCases(t *testing.T) {
|
||||
for _, testCase := range localBackendTestCases {
|
||||
// Clear global state left behind by tests that chdir to a temp directory.
|
||||
cache.Clear()
|
||||
|
||||
// TODO: Remove this when rclone requires a Go version >= 1.22. Future
|
||||
// versions of Go fix the semantics of capturing a range variable.
|
||||
// https://go.dev/blog/loopvar-preview
|
||||
testCase := testCase
|
||||
|
||||
t.Run(testCase.label, func(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
// Create temp dir for an rclone remote pointing at local filesystem.
|
||||
localFsDir := filepath.Join(tempDir, "remoteTarget")
|
||||
require.NoError(t, os.Mkdir(localFsDir, 0700))
|
||||
|
||||
// Create temp config
|
||||
remoteName := getUniqueRemoteName(t)
|
||||
configLines := []string{
|
||||
fmt.Sprintf("[%s]", remoteName),
|
||||
"type = local",
|
||||
fmt.Sprintf("remote = %s", localFsDir),
|
||||
}
|
||||
configContents := strings.Join(configLines, "\n")
|
||||
|
||||
configPath := filepath.Join(tempDir, "rclone.conf")
|
||||
require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
|
||||
require.NoError(t, config.SetConfigPath(configPath))
|
||||
|
||||
// The custom config file will be ignored unless we install the
|
||||
// global config file handler.
|
||||
configfile.Install()
|
||||
|
||||
handle := makeTestState(t)
|
||||
handle.localFsDir = localFsDir
|
||||
handle.configPath = configPath
|
||||
handle.remoteName = remoteName
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
|
||||
go func() {
|
||||
err := handle.server.run()
|
||||
|
||||
if testCase.expectedError == "" {
|
||||
require.NoError(t, err)
|
||||
} else {
|
||||
require.ErrorContains(t, err, testCase.expectedError)
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
}()
|
||||
defer wg.Wait()
|
||||
|
||||
testCase.testProtocolFunc(t, &handle)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Configure the git-annex client with a mockfs backend and send it the
|
||||
// "INITREMOTE" command over mocked stdin. This should fail because mockfs does
|
||||
// not support empty directories.
|
||||
func TestGitAnnexHandleInitRemoteBackendDoesNotSupportEmptyDirectories(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
// Temporarily override the filesystem registry.
|
||||
oldRegistry := fs.Registry
|
||||
mockfs.Register()
|
||||
defer func() { fs.Registry = oldRegistry }()
|
||||
|
||||
// Create temp dir for an rclone remote pointing at local filesystem.
|
||||
localFsDir := filepath.Join(tempDir, "remoteTarget")
|
||||
require.NoError(t, os.Mkdir(localFsDir, 0700))
|
||||
|
||||
// Create temp config
|
||||
remoteName := getUniqueRemoteName(t)
|
||||
configLines := []string{
|
||||
fmt.Sprintf("[%s]", remoteName),
|
||||
"type = mockfs",
|
||||
fmt.Sprintf("remote = %s", localFsDir),
|
||||
}
|
||||
configContents := strings.Join(configLines, "\n")
|
||||
|
||||
configPath := filepath.Join(tempDir, "rclone.conf")
|
||||
require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
|
||||
|
||||
// The custom config file will be ignored unless we install the global
|
||||
// config file handler.
|
||||
configfile.Install()
|
||||
require.NoError(t, config.SetConfigPath(configPath))
|
||||
|
||||
handle := makeTestState(t)
|
||||
handle.server.configPrefix = localFsDir
|
||||
handle.server.configRcloneRemoteName = remoteName
|
||||
handle.server.configsDone = true
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
|
||||
go func() {
|
||||
require.NotNil(t, handle.server.run())
|
||||
wg.Done()
|
||||
}()
|
||||
defer wg.Wait()
|
||||
|
||||
handle.requireReadLineExact("VERSION 2")
|
||||
handle.requireWriteLine("INITREMOTE")
|
||||
handle.requireReadLineExact("INITREMOTE-FAILURE this rclone remote does not support empty directories")
|
||||
}
|
||||
52
cmd/gitannex/run-git-annex-testremote.sh
Executable file
52
cmd/gitannex/run-git-annex-testremote.sh
Executable file
@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# End-to-end tests for "rclone gitannex". This script runs the `git-annex
|
||||
# testremote` suite against "rclone gitannex" in an ephemeral git-annex repo.
|
||||
#
|
||||
# Assumptions:
|
||||
#
|
||||
# * This system has an rclone remote configured named "git-annex-builtin-test-remote".
|
||||
#
|
||||
# * If it uses rclone's "local" backend, /tmp/git-annex-builtin-test-remote exists.
|
||||
|
||||
set -e
|
||||
|
||||
TEST_DIR="$(realpath "$(mktemp -d)")"
|
||||
mkdir "$TEST_DIR/bin"
|
||||
|
||||
function cleanup()
|
||||
{
|
||||
rm -rf "$TEST_DIR"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
RCLONE_DIR="$(git rev-parse --show-toplevel)"
|
||||
|
||||
rm -rf /tmp/git-annex-builtin-test-remote/*
|
||||
|
||||
set -x
|
||||
|
||||
pushd "$RCLONE_DIR"
|
||||
go build -o "$TEST_DIR/bin" ./
|
||||
|
||||
ln -s "$(realpath "$TEST_DIR/bin/rclone")" "$TEST_DIR/bin/git-annex-remote-rclone-builtin"
|
||||
popd
|
||||
|
||||
pushd "$TEST_DIR"
|
||||
|
||||
git init
|
||||
git annex init
|
||||
|
||||
REMOTE_NAME=git-annex-builtin-test-remote
|
||||
PREFIX=/tmp/git-annex-builtin-test-remote
|
||||
|
||||
PATH="$PATH:$TEST_DIR/bin" git annex initremote $REMOTE_NAME \
|
||||
type=external externaltype=rclone-builtin encryption=none \
|
||||
rcloneremotename=$REMOTE_NAME \
|
||||
rcloneprefix="$PREFIX"
|
||||
|
||||
PATH="$PATH:$(realpath bin)" git annex testremote $REMOTE_NAME
|
||||
|
||||
popd
|
||||
rm -rf "$TEST_DIR"
|
||||
Loading…
Reference in New Issue
Block a user