extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-22 16:34:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

onedrive: use lib/encoder

Browse Source
This commit is contained in:
Fabian Möller 2018-11-02 13:14:19 +01:00 committed by Nick Craig-Wood
parent d211347d46
commit dac20093c5
4 changed files with 67 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
backend/onedrive/onedrive.go
View File

@ -15,8 +15,6 @@ import (
"strings"
"time"
"github.com/rclone/rclone/lib/atexit"
"github.com/pkg/errors"
"github.com/rclone/rclone/backend/onedrive/api"
"github.com/rclone/rclone/fs"
@ -24,8 +22,10 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/encodings"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
@ -34,6 +34,8 @@ import (
"golang.org/x/oauth2"
)
const enc = encodings.OneDrive
const (
rcloneClientID = "b15665d9-eda6-4092-8539-0eec376afd59"
rcloneEncryptedClientSecret = "_JUdzh3LnKNqSPcf4Wu5fgMFIQOI8glZu_akYgR8yf6egowNBg-R"
@ -345,7 +347,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
// "shared with me" folders in OneDrive Personal (See #2536, #2778)
// This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480
func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath))))
opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(enc.FromStandardPath(relPath))))
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(resp, err)
@ -368,7 +370,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
} else {
opts = rest.Opts{
Method: "GET",
Path: "/root:/" + rest.URLPathEscape(replaceReservedChars(path)),
Path: "/root:/" + rest.URLPathEscape(enc.FromStandardPath(path)),
}
}
err = f.pacer.Call(func() (bool, error) {
@ -616,7 +618,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e
var info *api.Item
opts := newOptsCall(dirID, "POST", "/children")
mkdir := api.CreateItemRequest{
Name: replaceReservedChars(leaf),
Name: enc.FromStandardName(leaf),
ConflictBehavior: "fail",
}
err = f.pacer.Call(func() (bool, error) {
@ -676,7 +678,7 @@ OUTER:
if item.Deleted != nil {
continue
}
item.Name = restoreReservedChars(item.GetName())
item.Name = enc.ToStandardName(item.GetName())
if fn(item) {
found = true
break OUTER
@ -913,8 +915,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
return nil, err
}
srcPath := srcObj.fs.rootSlash() + srcObj.remote
dstPath := f.rootSlash() + remote
srcPath := srcObj.rootPath()
dstPath := f.rootPath(remote)
if strings.ToLower(srcPath) == strings.ToLower(dstPath) {
return nil, errors.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
}
@ -932,7 +934,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
id, dstDriveID, _ := parseNormalizedID(directoryID)
replacedLeaf := replaceReservedChars(leaf)
replacedLeaf := enc.FromStandardName(leaf)
copyReq := api.CopyItemRequest{
Name: &replacedLeaf,
ParentReference: api.ItemReference{
@ -1016,7 +1018,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
opts := newOptsCall(srcObj.id, "PATCH", "")
move := api.MoveItemRequest{
Name: replaceReservedChars(leaf),
Name: enc.FromStandardName(leaf),
ParentReference: &api.ItemReference{
DriveID: dstDriveID,
ID: id,
@ -1131,7 +1133,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
// Do the move
opts := newOptsCall(srcID, "PATCH", "")
move := api.MoveItemRequest{
Name: replaceReservedChars(leaf),
Name: enc.FromStandardName(leaf),
ParentReference: &api.ItemReference{
DriveID: dstDriveID,
ID: parsedDstDirID,
@ -1197,7 +1199,7 @@ func (f *Fs) Hashes() hash.Set {
// PublicLink returns a link for downloading without accout.
func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
info, _, err := f.readMetaDataForPath(ctx, f.srvPath(remote))
info, _, err := f.readMetaDataForPath(ctx, f.rootPath(remote))
if err != nil {
return "", err
}
@ -1241,9 +1243,19 @@ func (o *Object) Remote() string {
return o.remote
}
// rootPath returns a path for use in server given a remote
func (f *Fs) rootPath(remote string) string {
return f.rootSlash() + remote
}
// rootPath returns a path for use in local functions
func (o *Object) rootPath() string {
return o.fs.rootPath(o.remote)
}
// srvPath returns a path for use in server given a remote
func (f *Fs) srvPath(remote string) string {
return replaceReservedChars(f.rootSlash() + remote)
return enc.FromStandardPath(f.rootSlash() + remote)
}
// srvPath returns a path for use in server
@ -1320,7 +1332,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
if o.hasMetaData {
return nil
}
info, _, err := o.fs.readMetaDataForPath(ctx, o.srvPath())
info, _, err := o.fs.readMetaDataForPath(ctx, o.rootPath())
if err != nil {
if apiErr, ok := err.(*api.Error); ok {
if apiErr.ErrorInfo.Code == "itemNotFound" {
@ -1355,7 +1367,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
opts = rest.Opts{
Method: "PATCH",
RootURL: rootURL,
Path: "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(leaf)),
Path: "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(enc.FromStandardName(leaf))),
}
} else {
opts = rest.Opts{
@ -1429,7 +1441,8 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re
opts = rest.Opts{
Method: "POST",
RootURL: rootURL,
Path: "/" + drive + "/items/" + id + ":/" + rest.URLPathEscape(replaceReservedChars(leaf)) + ":/createUploadSession",
Path: fmt.Sprintf("/%s/items/%s:/%s:/createUploadSession",
drive, id, rest.URLPathEscape(enc.FromStandardName(leaf))),
}
} else {
opts = rest.Opts{
@ -1581,7 +1594,7 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
opts = rest.Opts{
Method: "PUT",
RootURL: rootURL,
Path: "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf) + ":/content",
Path: "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(enc.FromStandardName(leaf)) + ":/content",
ContentLength: &size,
Body: in,
}

91
backend/onedrive/replace.go
View File

@ -1,91 +0,0 @@
/*
Translate file names for one drive
OneDrive reserved characters
The following characters are OneDrive reserved characters, and can't
be used in OneDrive folder and file names.
onedrive-reserved = "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|"
onedrive-business-reserved
= "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|" / "#" / "%"
Note: Folder names can't end with a period (.).
Note: OneDrive for Business file or folder names cannot begin with a
tilde ('~').
*/
package onedrive
import (
"regexp"
"strings"
)
// charMap holds replacements for characters
//
// Onedrive has a restricted set of characters compared to other cloud
// storage systems, so we to map these to the FULLWIDTH unicode
// equivalents
//
// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
var (
charMap = map[rune]rune{
'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
'*': '＊', // FULLWIDTH ASTERISK
'<': '＜', // FULLWIDTH LESS-THAN SIGN
'>': '＞', // FULLWIDTH GREATER-THAN SIGN
'?': '？', // FULLWIDTH QUESTION MARK
':': '：', // FULLWIDTH COLON
'|': '｜', // FULLWIDTH VERTICAL LINE
'#': '＃', // FULLWIDTH NUMBER SIGN
'%': '％', // FULLWIDTH PERCENT SIGN
'"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
'.': '．', // FULLWIDTH FULL STOP
'~': '～', // FULLWIDTH TILDE
' ': '␠', // SYMBOL FOR SPACE
}
invCharMap map[rune]rune
fixEndingInPeriod = regexp.MustCompile(`\.(/|$)`)
fixStartingWithTilde = regexp.MustCompile(`(/|^)~`)
fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
)
func init() {
// Create inverse charMap
invCharMap = make(map[rune]rune, len(charMap))
for k, v := range charMap {
invCharMap[v] = k
}
}
// replaceReservedChars takes a path and substitutes any reserved
// characters in it
func replaceReservedChars(in string) string {
// Folder names can't end with a period '.'
in = fixEndingInPeriod.ReplaceAllString(in, string(charMap['.'])+"$1")
// OneDrive for Business file or folder names cannot begin with a tilde '~'
in = fixStartingWithTilde.ReplaceAllString(in, "$1"+string(charMap['~']))
// Apparently file names can't start with space either
in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
// Replace reserved characters
return strings.Map(func(c rune) rune {
if replacement, ok := charMap[c]; ok && c != '.' && c != '~' && c != ' ' {
return replacement
}
return c
}, in)
}
// restoreReservedChars takes a path and undoes any substitutions
// made by replaceReservedChars
func restoreReservedChars(in string) string {
return strings.Map(func(c rune) rune {
if replacement, ok := invCharMap[c]; ok {
return replacement
}
return c
}, in)
}

30
backend/onedrive/replace_test.go
View File

@ -1,30 +0,0 @@
package onedrive
import "testing"
func TestReplace(t *testing.T) {
for _, test := range []struct {
in string
out string
}{
{"", ""},
{"abc 123", "abc 123"},
{`\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~`},
{`\*<>?:|#%".~/\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~/＼＊＜＞？：｜＃％＂.~`},
{" leading space", "␠leading space"},
{"~leading tilde", "～leading tilde"},
{"trailing dot.", "trailing dot．"},
{" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"},
{"~leading tilde/~leading tilde/~leading tilde", "～leading tilde/～leading tilde/～leading tilde"},
{"trailing dot./trailing dot./trailing dot.", "trailing dot．/trailing dot．/trailing dot．"},
} {
got := replaceReservedChars(test.in)
if got != test.out {
t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
}
got2 := restoreReservedChars(got)
if got2 != test.in {
t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
}
}
}

37
docs/content/onedrive.md
View File

@ -148,6 +148,43 @@ Sharepoint Server support
For all types of OneDrive you can use the `--checksum` flag.
#### Restricted filename characters
In addition to the [default restricted characters set](/overview/#restricted-characters)
the following characters are also replaced:
| Character | Value | Replacement |
| --------- |:-----:|:-----------:|
| " | 0x22 | ＂ |
| * | 0x2A | ＊ |
| : | 0x3A | ： |
| < | 0x3C | ＜ |
| > | 0x3E | ＞ |
| ? | 0x3F | ？ |
| \ | 0x5C | ＼ |
| \| | 0x7C | ｜ |
| # | 0x23 | ＃ |
| % | 0x25 | ％ |
File names can also not end with the following characters.
These only get replaced if they are last character in the name:
| Character | Value | Replacement |
| --------- |:-----:|:-----------:|
| SP | 0x20 | ␠ |
| . | 0x2E | ． |
File names can also not begin with the following characters.
These only get replaced if they are first character in the name:
| Character | Value | Replacement |
| --------- |:-----:|:-----------:|
| SP | 0x20 | ␠ |
| ~ | 0x7E | ～ |
Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
as they can't be used in JSON strings.
### Deleting files ###
Any files you delete with rclone will end up in the trash. Microsoft