mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 00:13:49 +01:00
cmd/gitannex: Add support for different layouts
This commit adds support for the same repo layouts supported by git-annex-remote-rclone. This should enable git-annex users with remotes of type "rclone" to switch to a "rclone-builtin" without needing to retransfer content. Issue #7625
This commit is contained in:
parent
36ad4eb145
commit
29b58dd4c5
@ -110,9 +110,10 @@ func (m *messageParser) finalParameter() string {
|
||||
// configDefinition describes a configuration value required by this command. We
|
||||
// use "GETCONFIG" messages to query git-annex for these values at runtime.
|
||||
type configDefinition struct {
|
||||
name string
|
||||
description string
|
||||
destination *string
|
||||
name string
|
||||
description string
|
||||
destination *string
|
||||
defaultValue *string
|
||||
}
|
||||
|
||||
// server contains this command's current state.
|
||||
@ -132,6 +133,7 @@ type server struct {
|
||||
configsDone bool
|
||||
configPrefix string
|
||||
configRcloneRemoteName string
|
||||
configRcloneLayout string
|
||||
}
|
||||
|
||||
func (s *server) sendMsg(msg string) {
|
||||
@ -267,6 +269,9 @@ func (s *server) handleInitRemote() error {
|
||||
|
||||
// Get a list of configs with pointers to fields of `s`.
|
||||
func (s *server) getRequiredConfigs() []configDefinition {
|
||||
defaultRclonePrefix := "git-annex-rclone"
|
||||
defaultRcloneLayout := "nodir"
|
||||
|
||||
return []configDefinition{
|
||||
{
|
||||
"rcloneremotename",
|
||||
@ -274,13 +279,23 @@ func (s *server) getRequiredConfigs() []configDefinition {
|
||||
"Must match a remote known to rclone. " +
|
||||
"(Note that rclone remotes are a distinct concept from git-annex remotes.)",
|
||||
&s.configRcloneRemoteName,
|
||||
nil,
|
||||
},
|
||||
{
|
||||
"rcloneprefix",
|
||||
"Directory where rclone will write git-annex content. " +
|
||||
"If not specified, defaults to \"git-annex-rclone\". " +
|
||||
"This directory be created on init if it does not exist.",
|
||||
fmt.Sprintf("If not specified, defaults to %q. ", defaultRclonePrefix) +
|
||||
"This directory will be created on init if it does not exist.",
|
||||
&s.configPrefix,
|
||||
&defaultRclonePrefix,
|
||||
},
|
||||
{
|
||||
"rclonelayout",
|
||||
"Defines where, within the rcloneprefix directory, rclone will write git-annex content. " +
|
||||
fmt.Sprintf("Must be one of %v. ", allLayoutModes()) +
|
||||
fmt.Sprintf("If empty, defaults to %q.", defaultRcloneLayout),
|
||||
&s.configRcloneLayout,
|
||||
&defaultRcloneLayout,
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -307,10 +322,13 @@ func (s *server) queryConfigs() error {
|
||||
}
|
||||
|
||||
value := message.finalParameter()
|
||||
if value == "" {
|
||||
if value == "" && config.defaultValue == nil {
|
||||
return fmt.Errorf("config value of %q must not be empty", config.name)
|
||||
}
|
||||
|
||||
if value == "" {
|
||||
*config.destination = *config.defaultValue
|
||||
continue
|
||||
}
|
||||
*config.destination = value
|
||||
}
|
||||
|
||||
@ -358,7 +376,19 @@ func (s *server) handleTransfer(message *messageParser) error {
|
||||
return fmt.Errorf("error getting configs: %w", err)
|
||||
}
|
||||
|
||||
remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
|
||||
layout := parseLayoutMode(s.configRcloneLayout)
|
||||
if layout == layoutModeUnknown {
|
||||
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey))
|
||||
return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
|
||||
}
|
||||
|
||||
remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey))
|
||||
return fmt.Errorf("error building fs string: %w", err)
|
||||
}
|
||||
|
||||
remoteFs, err := cache.Get(context.TODO(), remoteFsString)
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey))
|
||||
return err
|
||||
@ -415,7 +445,19 @@ func (s *server) handleCheckPresent(message *messageParser) error {
|
||||
return fmt.Errorf("error getting configs: %s", err)
|
||||
}
|
||||
|
||||
remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
|
||||
layout := parseLayoutMode(s.configRcloneLayout)
|
||||
if layout == layoutModeUnknown {
|
||||
s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))
|
||||
return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
|
||||
}
|
||||
|
||||
remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))
|
||||
return fmt.Errorf("error building fs string: %w", err)
|
||||
}
|
||||
|
||||
remoteFs, err := cache.Get(context.TODO(), remoteFsString)
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey))
|
||||
return err
|
||||
@ -435,17 +477,45 @@ func (s *server) handleCheckPresent(message *messageParser) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *server) queryDirhash(msg string) (string, error) {
|
||||
s.sendMsg(msg)
|
||||
parser, err := s.getMsg()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
keyword, err := parser.nextSpaceDelimitedParameter()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if keyword != "VALUE" {
|
||||
return "", fmt.Errorf("expected VALUE keyword, but got %q", keyword)
|
||||
}
|
||||
dirhash, err := parser.nextSpaceDelimitedParameter()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse dirhash: %w", err)
|
||||
}
|
||||
return dirhash, nil
|
||||
}
|
||||
|
||||
func (s *server) handleRemove(message *messageParser) error {
|
||||
argKey := message.finalParameter()
|
||||
if argKey == "" {
|
||||
return errors.New("failed to parse key for REMOVE")
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
layout := parseLayoutMode(s.configRcloneLayout)
|
||||
if layout == layoutModeUnknown {
|
||||
s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
|
||||
return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
|
||||
}
|
||||
|
||||
remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
|
||||
remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
|
||||
return fmt.Errorf("error building fs string: %w", err)
|
||||
}
|
||||
|
||||
remoteFs, err := cache.Get(context.TODO(), remoteFsString)
|
||||
if err != nil {
|
||||
s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
|
||||
return fmt.Errorf("error getting remote fs: %w", err)
|
||||
|
@ -234,6 +234,7 @@ func (h *testState) requireWriteLine(line string) {
|
||||
func (h *testState) preconfigureServer() {
|
||||
h.server.configPrefix = h.localFsDir
|
||||
h.server.configRcloneRemoteName = h.remoteName
|
||||
h.server.configRcloneLayout = string(layoutModeNodir)
|
||||
h.server.configsDone = true
|
||||
}
|
||||
|
||||
@ -285,6 +286,8 @@ var localBackendTestCases = []testCase{
|
||||
h.requireWriteLine("VALUE " + h.remoteName)
|
||||
h.requireReadLineExact("GETCONFIG rcloneprefix")
|
||||
h.requireWriteLine("VALUE " + h.localFsDir)
|
||||
h.requireReadLineExact("GETCONFIG rclonelayout")
|
||||
h.requireWriteLine("VALUE foo")
|
||||
h.requireReadLineExact("PREPARE-SUCCESS")
|
||||
|
||||
require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
|
||||
@ -313,9 +316,13 @@ var localBackendTestCases = []testCase{
|
||||
localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)
|
||||
|
||||
h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))
|
||||
h.requireReadLineExact("GETCONFIG rcloneprefix")
|
||||
|
||||
h.requireReadLineExact("GETCONFIG rcloneprefix")
|
||||
h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))
|
||||
|
||||
h.requireReadLineExact("GETCONFIG rclonelayout")
|
||||
h.requireWriteLine("VALUE")
|
||||
|
||||
h.requireReadLineExact("PREPARE-SUCCESS")
|
||||
|
||||
require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)
|
||||
@ -367,11 +374,11 @@ var localBackendTestCases = []testCase{
|
||||
|
||||
// Note the whitespace following the key.
|
||||
h.requireWriteLine("TRANSFER STORE Key ")
|
||||
h.requireReadLineExact("TRANSFER-FAILURE failed to parse file")
|
||||
h.requireReadLineExact("TRANSFER-FAILURE failed to parse file path")
|
||||
|
||||
require.NoError(t, h.mockStdinW.Close())
|
||||
},
|
||||
expectedError: "malformed arguments for TRANSFER: nothing remains to parse",
|
||||
expectedError: "failed to parse file",
|
||||
},
|
||||
// Repeated EXTENSIONS messages add to each other rather than overriding
|
||||
// prior advertised extensions. This behavior is not mandated by the
|
||||
|
72
cmd/gitannex/layout.go
Normal file
72
cmd/gitannex/layout.go
Normal file
@ -0,0 +1,72 @@
|
||||
package gitannex
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type layoutMode string
|
||||
|
||||
// All layout modes from git-annex-remote-rclone are supported.
|
||||
const (
|
||||
layoutModeLower layoutMode = "lower"
|
||||
layoutModeDirectory layoutMode = "directory"
|
||||
layoutModeNodir layoutMode = "nodir"
|
||||
layoutModeMixed layoutMode = "mixed"
|
||||
layoutModeFrankencase layoutMode = "frankencase"
|
||||
layoutModeUnknown layoutMode = ""
|
||||
)
|
||||
|
||||
func allLayoutModes() []layoutMode {
|
||||
return []layoutMode{
|
||||
layoutModeLower,
|
||||
layoutModeDirectory,
|
||||
layoutModeNodir,
|
||||
layoutModeMixed,
|
||||
layoutModeFrankencase,
|
||||
}
|
||||
}
|
||||
|
||||
func parseLayoutMode(mode string) layoutMode {
|
||||
for _, knownMode := range allLayoutModes() {
|
||||
if mode == string(knownMode) {
|
||||
return knownMode
|
||||
}
|
||||
}
|
||||
return layoutModeUnknown
|
||||
}
|
||||
|
||||
type queryDirhashFunc func(msg string) (string, error)
|
||||
|
||||
func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteName, prefix string) (string, error) {
|
||||
if mode == layoutModeNodir {
|
||||
return fmt.Sprintf("%s:%s", remoteName, prefix), nil
|
||||
}
|
||||
|
||||
var dirhash string
|
||||
var err error
|
||||
switch mode {
|
||||
case layoutModeLower, layoutModeDirectory:
|
||||
dirhash, err = queryDirhash("DIRHASH-LOWER " + key)
|
||||
case layoutModeMixed, layoutModeFrankencase:
|
||||
dirhash, err = queryDirhash("DIRHASH " + key)
|
||||
default:
|
||||
panic("unreachable")
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("buildFsString failed to query dirhash: %w", err)
|
||||
}
|
||||
|
||||
switch mode {
|
||||
case layoutModeLower:
|
||||
return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
|
||||
case layoutModeDirectory:
|
||||
return fmt.Sprintf("%s:%s/%s%s", remoteName, prefix, dirhash, key), nil
|
||||
case layoutModeMixed:
|
||||
return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
|
||||
case layoutModeFrankencase:
|
||||
return fmt.Sprintf("%s:%s/%s", remoteName, prefix, strings.ToLower(dirhash)), nil
|
||||
default:
|
||||
panic("unreachable")
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user