From 4837bc354682283b4f351c80f8e157f6bd980400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Fri, 2 Nov 2018 13:13:47 +0100 Subject: [PATCH] jottacloud: use lib/encoder --- backend/jottacloud/jottacloud.go | 19 ++++---- backend/jottacloud/replace.go | 77 ------------------------------ backend/jottacloud/replace_test.go | 28 ----------- docs/content/jottacloud.md | 18 +++++++ fs/encodings/encodings.go | 3 ++ 5 files changed, 32 insertions(+), 113 deletions(-) delete mode 100644 backend/jottacloud/replace.go delete mode 100644 backend/jottacloud/replace_test.go diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 0d365230b..36e18ebfa 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -26,6 +26,7 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -36,6 +37,8 @@ import ( "golang.org/x/oauth2" ) +const enc = encodings.JottaCloud + // Globals const ( minSleep = 10 * time.Millisecond @@ -460,7 +463,7 @@ func urlPathEscape(in string) string { // filePathRaw returns an unescaped file path (f.root, file) func (f *Fs) filePathRaw(file string) string { - return path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, file))) + return path.Join(f.endpointURL, enc.FromStandardPath(path.Join(f.root, file))) } // filePath returns a escaped file path (f.root, file) @@ -673,7 +676,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e if item.Deleted { continue } - remote := path.Join(dir, restoreReservedChars(item.Name)) + remote := path.Join(dir, enc.ToStandardName(item.Name)) d := fs.NewDir(remote, time.Time(item.ModifiedAt)) entries = append(entries, d) } @@ -683,7 +686,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e if item.Deleted || item.State != "COMPLETED" { continue } - remote := path.Join(dir, restoreReservedChars(item.Name)) + remote := path.Join(dir, enc.ToStandardName(item.Name)) o, err := f.newObjectWithInfo(ctx, remote, item) if err != nil { continue @@ -708,7 +711,7 @@ func (f *Fs) listFileDir(ctx context.Context, remoteStartPath string, startFolde if folder.Deleted { return nil } - folderPath := restoreReservedChars(path.Join(folder.Path, folder.Name)) + folderPath := enc.ToStandardPath(path.Join(folder.Path, folder.Name)) folderPathLength := len(folderPath) var remoteDir string if folderPathLength > pathPrefixLength { @@ -726,7 +729,7 @@ func (f *Fs) listFileDir(ctx context.Context, remoteStartPath string, startFolde if file.Deleted || file.State != "COMPLETED" { continue } - remoteFile := path.Join(remoteDir, restoreReservedChars(file.Name)) + remoteFile := path.Join(remoteDir, enc.ToStandardName(file.Name)) o, err := f.newObjectWithInfo(ctx, remoteFile, file) if err != nil { return err @@ -897,7 +900,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap Parameters: url.Values{}, } - opts.Parameters.Set(method, "/"+path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, dest)))) + opts.Parameters.Set(method, "/"+path.Join(f.endpointURL, enc.FromStandardPath(path.Join(f.root, dest)))) var resp *http.Response err = f.pacer.Call(func() (bool, error) { @@ -1004,7 +1007,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string return fs.ErrorDirExists } - _, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, replaceReservedChars(srcPath))+"/", dstRemote) + _, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, enc.FromStandardPath(srcPath))+"/", dstRemote) if err != nil { return errors.Wrap(err, "couldn't move directory") @@ -1295,7 +1298,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op Created: fileDate, Modified: fileDate, Md5: md5String, - Path: path.Join(o.fs.opt.Mountpoint, replaceReservedChars(path.Join(o.fs.root, o.remote))), + Path: path.Join(o.fs.opt.Mountpoint, enc.FromStandardPath(path.Join(o.fs.root, o.remote))), } // send it diff --git a/backend/jottacloud/replace.go b/backend/jottacloud/replace.go deleted file mode 100644 index 698726036..000000000 --- a/backend/jottacloud/replace.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Translate file names for JottaCloud adapted from OneDrive - - -The following characters are JottaCloud reserved characters, and can't -be used in JottaCloud folder and file names. - - jottacloud = "/" / "\" / "*" / "<" / ">" / "?" / "!" / "&" / ":" / ";" / "|" / "#" / "%" / """ / "'" / "." / "~" - - -*/ - -package jottacloud - -import ( - "regexp" - "strings" -) - -// charMap holds replacements for characters -// -// Onedrive has a restricted set of characters compared to other cloud -// storage systems, so we to map these to the FULLWIDTH unicode -// equivalents -// -// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS -var ( - charMap = map[rune]rune{ - '\\': '＼', // FULLWIDTH REVERSE SOLIDUS - '*': '＊', // FULLWIDTH ASTERISK - '<': '＜', // FULLWIDTH LESS-THAN SIGN - '>': '＞', // FULLWIDTH GREATER-THAN SIGN - '?': '？', // FULLWIDTH QUESTION MARK - ':': '：', // FULLWIDTH COLON - ';': '；', // FULLWIDTH SEMICOLON - '|': '｜', // FULLWIDTH VERTICAL LINE - '"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved - ' ': '␠', // SYMBOL FOR SPACE - } - invCharMap map[rune]rune - fixStartingWithSpace = regexp.MustCompile(`(/|^) `) - fixEndingWithSpace = regexp.MustCompile(` (/|$)`) -) - -func init() { - // Create inverse charMap - invCharMap = make(map[rune]rune, len(charMap)) - for k, v := range charMap { - invCharMap[v] = k - } -} - -// replaceReservedChars takes a path and substitutes any reserved -// characters in it -func replaceReservedChars(in string) string { - // Filenames can't start with space - in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' '])) - // Filenames can't end with space - in = fixEndingWithSpace.ReplaceAllString(in, string(charMap[' '])+"$1") - return strings.Map(func(c rune) rune { - if replacement, ok := charMap[c]; ok && c != ' ' { - return replacement - } - return c - }, in) -} - -// restoreReservedChars takes a path and undoes any substitutions -// made by replaceReservedChars -func restoreReservedChars(in string) string { - return strings.Map(func(c rune) rune { - if replacement, ok := invCharMap[c]; ok { - return replacement - } - return c - }, in) -} diff --git a/backend/jottacloud/replace_test.go b/backend/jottacloud/replace_test.go deleted file mode 100644 index b1f2979b8..000000000 --- a/backend/jottacloud/replace_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package jottacloud - -import "testing" - -func TestReplace(t *testing.T) { - for _, test := range []struct { - in string - out string - }{ - {"", ""}, - {"abc 123", "abc 123"}, - {`\*<>?:;|"`, `＼＊＜＞？：；｜＂`}, - {`\*<>?:;|"\*<>?:;|"`, `＼＊＜＞？：；｜＂＼＊＜＞？：；｜＂`}, - {" leading space", "␠leading space"}, - {"trailing space ", "trailing space␠"}, - {" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"}, - {"trailing space /trailing space /trailing space ", "trailing space␠/trailing space␠/trailing space␠"}, - } { - got := replaceReservedChars(test.in) - if got != test.out { - t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got) - } - got2 := restoreReservedChars(got) - if got2 != test.in { - t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2) - } - } -} diff --git a/docs/content/jottacloud.md b/docs/content/jottacloud.md index 4a5df99ec..e6c7a1f2e 100644 --- a/docs/content/jottacloud.md +++ b/docs/content/jottacloud.md @@ -129,6 +129,24 @@ temporarily on disk (wherever the `TMPDIR` environment variable points to) before it is uploaded. Small files will be cached in memory - see the `--jottacloud-md5-memory-limit` flag. +#### Restricted filename characters + +In addition to the [default restricted characters set](/overview/#restricted-characters) +the following characters are also replaced: + +| Character | Value | Replacement | +| --------- |:-----:|:-----------:| +| " | 0x22 | ＂ | +| * | 0x2A | ＊ | +| : | 0x3A | ： | +| < | 0x3C | ＜ | +| > | 0x3E | ＞ | +| ? | 0x3F | ？ | +| \| | 0x7C | ｜ | + +Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8), +as they can't be used in XML strings. + ### Deleting files ### By default rclone will send all files to the trash when deleting files. diff --git a/fs/encodings/encodings.go b/fs/encodings/encodings.go index fefac77d4..f230c285b 100644 --- a/fs/encodings/encodings.go +++ b/fs/encodings/encodings.go @@ -110,8 +110,11 @@ const GoogleCloudStorage = encoder.MultiEncoder( // JottaCloud is the encoding used by the jottacloud backend // // Encode invalid UTF-8 bytes as xml doesn't handle them properly. +// +// Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|' const JottaCloud = encoder.MultiEncoder( uint(Display) | + encoder.EncodeWin | // :?"*<>| encoder.EncodeInvalidUtf8) // Koofr is the encoding used by the koofr backend