From bafe7d5a73f1b77ab30edae62650132610926b03 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 14 Jan 2020 21:51:49 +0000 Subject: [PATCH] backends: move encoding definitions from fs/encodings --- backend/amazonclouddrive/amazonclouddrive.go | 5 +- backend/azureblob/azureblob.go | 8 +- backend/b2/b2.go | 8 +- backend/box/box.go | 13 +- backend/drive/drive.go | 5 +- backend/dropbox/dropbox.go | 11 +- backend/fichier/fichier.go | 23 +- backend/ftp/ftp.go | 9 +- .../googlecloudstorage/googlecloudstorage.go | 5 +- backend/jottacloud/jottacloud.go | 8 +- backend/koofr/koofr.go | 6 +- backend/local/encode_darwin.go | 10 +- backend/local/encode_other.go | 7 +- backend/local/encode_windows.go | 32 +- backend/mailru/mailru.go | 7 +- backend/mega/mega.go | 5 +- backend/onedrive/onedrive.go | 44 +- backend/opendrive/opendrive.go | 31 +- backend/pcloud/pcloud.go | 8 +- backend/premiumizeme/premiumizeme.go | 7 +- backend/putio/putio.go | 8 +- backend/qingstor/qingstor.go | 5 +- backend/s3/s3.go | 15 +- backend/sharefile/sharefile.go | 11 +- backend/swift/swift.go | 4 +- backend/yandex/yandex.go | 6 +- fs/encodings/encodings.go | 419 ------------------ fs/encodings/encodings_noencode.go | 57 --- 28 files changed, 243 insertions(+), 534 deletions(-) delete mode 100644 fs/encodings/encodings.go delete mode 100644 fs/encodings/encodings_noencode.go diff --git a/backend/amazonclouddrive/amazonclouddrive.go b/backend/amazonclouddrive/amazonclouddrive.go index 45eefa046..7d026eb95 100644 --- a/backend/amazonclouddrive/amazonclouddrive.go +++ b/backend/amazonclouddrive/amazonclouddrive.go @@ -28,7 +28,6 @@ import ( "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -141,7 +140,9 @@ underlying S3 storage.`, Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.AmazonCloudDrive, + // Encode invalid UTF-8 bytes as json doesn't handle them properly. + Default: (encoder.Base | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index f770f4293..224c82e2b 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -29,7 +29,6 @@ import ( "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -131,7 +130,12 @@ tiering blob to "Hot" or "Cool".`, Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.AzureBlob, + Default: (encoder.EncodeInvalidUtf8 | + encoder.EncodeSlash | + encoder.EncodeCtl | + encoder.EncodeDel | + encoder.EncodeBackSlash | + encoder.EncodeRightPeriod), }}, }) } diff --git a/backend/b2/b2.go b/backend/b2/b2.go index 123b37267..98212e27a 100644 --- a/backend/b2/b2.go +++ b/backend/b2/b2.go @@ -26,7 +26,6 @@ import ( "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -150,7 +149,12 @@ The minimum value is 1 second. The maximum value is one week.`, Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.B2, + // See: https://www.backblaze.com/b2/docs/files.html + // Encode invalid UTF-8 bytes as json doesn't handle them properly. + // FIXME: allow /, but not leading, trailing or double + Default: (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/box/box.go b/backend/box/box.go index 79d29b6d7..520119c6a 100644 --- a/backend/box/box.go +++ b/backend/box/box.go @@ -37,7 +37,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -149,7 +148,17 @@ func init() { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Box, + // From https://developer.box.com/docs/error-codes#section-400-bad-request : + // > Box only supports file or folder names that are 255 characters or less. + // > File names containing non-printable ascii, "/" or "\", names with leading + // > or trailing spaces, and the special names “.” and “..” are also unsupported. + // + // Testing revealed names with leading spaces work fine. + // Also encode invalid UTF-8 bytes as json doesn't handle them properly. + Default: (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeRightSpace | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 02e930c6d..6cc17e163 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -33,7 +33,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -459,7 +458,9 @@ See: https://github.com/rclone/rclone/issues/3857 Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Drive, + // Encode invalid UTF-8 bytes as json doesn't handle them properly. + // Don't encode / as it's a valid name character in drive. + Default: encoder.EncodeInvalidUtf8, }}, }) diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 340e0b8de..e39fa44b2 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -45,7 +45,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/encoder" @@ -150,7 +149,15 @@ memory. It can be set smaller if you are tight on memory.`, maxChunkSize), Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Dropbox, + // https://www.dropbox.com/help/syncing-uploads/files-not-syncing lists / and \ + // as invalid characters. + // Testing revealed names with trailing spaces and the DEL character don't work. + // Also encode invalid UTF-8 bytes as json doesn't handle them properly. + Default: (encoder.Base | + encoder.EncodeBackSlash | + encoder.EncodeDel | + encoder.EncodeRightSpace | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/fichier/fichier.go b/backend/fichier/fichier.go index e43f24377..f237af48e 100644 --- a/backend/fichier/fichier.go +++ b/backend/fichier/fichier.go @@ -14,7 +14,6 @@ import ( "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/dircache" @@ -50,7 +49,27 @@ func init() { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Fichier, + // Characters that need escaping + // + // '\\': '＼', // FULLWIDTH REVERSE SOLIDUS + // '<': '＜', // FULLWIDTH LESS-THAN SIGN + // '>': '＞', // FULLWIDTH GREATER-THAN SIGN + // '"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved + // '\'': '＇', // FULLWIDTH APOSTROPHE + // '$': '＄', // FULLWIDTH DOLLAR SIGN + // '`': '｀', // FULLWIDTH GRAVE ACCENT + // + // Leading space and trailing space + Default: (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeSingleQuote | + encoder.EncodeBackQuote | + encoder.EncodeDoubleQuote | + encoder.EncodeLtGt | + encoder.EncodeDollar | + encoder.EncodeLeftSpace | + encoder.EncodeRightSpace | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index 5a4966b84..7631209da 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -18,7 +18,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/pacer" @@ -73,7 +72,13 @@ func init() { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.FTP, + // The FTP protocal can't handle trailing spaces (for instance + // pureftpd turns them into _) + // + // proftpd can't handle '*' in file names + // pureftpd can't handle '[', ']' or '*' + Default: (encoder.Display | + encoder.EncodeRightSpace), }}, }) } diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index 88a7877f6..f74f7087b 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -32,7 +32,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -251,7 +250,9 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.GoogleCloudStorage, + Default: (encoder.Base | + encoder.EncodeCrLf | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index b510fc9fc..990dc36f8 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -26,7 +26,6 @@ import ( "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -160,7 +159,12 @@ func init() { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.JottaCloud, + // Encode invalid UTF-8 bytes as xml doesn't handle them properly. + // + // Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|' + Default: (encoder.Display | + encoder.EncodeWin | // :?"*<>| + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/koofr/koofr.go b/backend/koofr/koofr.go index d56783c3c..b97c0a9e4 100644 --- a/backend/koofr/koofr.go +++ b/backend/koofr/koofr.go @@ -16,7 +16,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/encoder" @@ -62,7 +61,10 @@ func init() { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Koofr, + // Encode invalid UTF-8 bytes as json doesn't handle them properly. + Default: (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/local/encode_darwin.go b/backend/local/encode_darwin.go index 6ba06cc07..e5615b318 100644 --- a/backend/local/encode_darwin.go +++ b/backend/local/encode_darwin.go @@ -2,8 +2,10 @@ package local -import ( - "github.com/rclone/rclone/fs/encodings" -) +import "github.com/rclone/rclone/lib/encoder" -const defaultEnc = encodings.LocalMacOS +// This is the encoding used by the local backend for macOS +// +// macOS can't store invalid UTF-8, it converts them into %XX encoding +const defaultEnc = (encoder.Base | + encoder.EncodeInvalidUtf8) diff --git a/backend/local/encode_other.go b/backend/local/encode_other.go index 848c8a2ae..2fcafe84e 100644 --- a/backend/local/encode_other.go +++ b/backend/local/encode_other.go @@ -2,8 +2,7 @@ package local -import ( - "github.com/rclone/rclone/fs/encodings" -) +import "github.com/rclone/rclone/lib/encoder" -const defaultEnc = encodings.LocalUnix +// This is the encoding used by the local backend for non windows platforms +const defaultEnc = encoder.Base diff --git a/backend/local/encode_windows.go b/backend/local/encode_windows.go index 86e7e5ca5..68e87a914 100644 --- a/backend/local/encode_windows.go +++ b/backend/local/encode_windows.go @@ -2,8 +2,32 @@ package local -import ( - "github.com/rclone/rclone/fs/encodings" -) +import "github.com/rclone/rclone/lib/encoder" -const defaultEnc = encodings.LocalWindows +// This is the encoding used by the local backend for windows platforms +// +// List of replaced characters: +// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN +// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN +// : (colon) -> '：' // FULLWIDTH COLON +// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK +// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS +// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE +// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK +// * (asterisk) -> '＊' // FULLWIDTH ASTERISK +// +// Additionally names can't end with a period (.) or space ( ). +// List of replaced characters: +// . (period) -> '．' // FULLWIDTH FULL STOP +// (space) -> '␠' // SYMBOL FOR SPACE +// +// Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16. +// +// https://docs.microsoft.com/de-de/windows/desktop/FileIO/naming-a-file#naming-conventions +const defaultEnc = (encoder.Base | + encoder.EncodeWin | + encoder.EncodeBackSlash | + encoder.EncodeCtl | + encoder.EncodeRightSpace | + encoder.EncodeRightPeriod | + encoder.EncodeInvalidUtf8) diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go index 00fc2890b..f163eb21a 100644 --- a/backend/mailru/mailru.go +++ b/backend/mailru/mailru.go @@ -28,7 +28,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -197,7 +196,11 @@ Supported quirks: atomicmkdir binlist gzip insecure retry400`, Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Mailru, + // Encode invalid UTF-8 bytes as json doesn't handle them properly. + Default: (encoder.Display | + encoder.EncodeWin | // :?"*<>| + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/mega/mega.go b/backend/mega/mega.go index 5209018b6..1802fa962 100644 --- a/backend/mega/mega.go +++ b/backend/mega/mega.go @@ -30,7 +30,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/encoder" @@ -87,7 +86,9 @@ permanently delete objects instead.`, Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Mega, + // Encode invalid UTF-8 bytes as json doesn't handle them properly. + Default: (encoder.Base | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index dd24f9750..23e5a04bf 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -23,7 +23,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/atexit" @@ -255,7 +254,48 @@ listing, set this option.`, Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.OneDrive, + // List of replaced characters: + // < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN + // > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN + // : (colon) -> '：' // FULLWIDTH COLON + // " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK + // \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS + // | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE + // ? (question mark) -> '？' // FULLWIDTH QUESTION MARK + // * (asterisk) -> '＊' // FULLWIDTH ASTERISK + // # (number sign) -> '＃' // FULLWIDTH NUMBER SIGN + // % (percent sign) -> '％' // FULLWIDTH PERCENT SIGN + // + // Folder names cannot begin with a tilde ('~') + // List of replaced characters: + // ~ (tilde) -> '～' // FULLWIDTH TILDE + // + // Additionally names can't begin with a space ( ) or end with a period (.) or space ( ). + // List of replaced characters: + // . (period) -> '．' // FULLWIDTH FULL STOP + // (space) -> '␠' // SYMBOL FOR SPACE + // + // Also encode invalid UTF-8 bytes as json doesn't handle them. + // + // The OneDrive API documentation lists the set of reserved characters, but + // testing showed this list is incomplete. This are the differences: + // - " (double quote) is rejected, but missing in the documentation + // - space at the end of file and folder names is rejected, but missing in the documentation + // - period at the end of file names is rejected, but missing in the documentation + // + // Adding these restrictions to the OneDrive API documentation yields exactly + // the same rules as the Windows naming conventions. + // + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online#path-encoding + Default: (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeHashPercent | + encoder.EncodeLeftSpace | + encoder.EncodeLeftTilde | + encoder.EncodeRightPeriod | + encoder.EncodeRightSpace | + encoder.EncodeWin | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.go index bdd004957..08b9b0dc2 100644 --- a/backend/opendrive/opendrive.go +++ b/backend/opendrive/opendrive.go @@ -17,7 +17,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -54,7 +53,35 @@ func init() { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.OpenDrive, + // List of replaced characters: + // < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN + // > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN + // : (colon) -> '：' // FULLWIDTH COLON + // " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK + // \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS + // | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE + // ? (question mark) -> '？' // FULLWIDTH QUESTION MARK + // * (asterisk) -> '＊' // FULLWIDTH ASTERISK + // + // Additionally names can't begin or end with a ASCII whitespace. + // List of replaced characters: + // (space) -> '␠' // SYMBOL FOR SPACE + // (horizontal tab) -> '␉' // SYMBOL FOR HORIZONTAL TABULATION + // (line feed) -> '␊' // SYMBOL FOR LINE FEED + // (vertical tab) -> '␋' // SYMBOL FOR VERTICAL TABULATION + // (carriage return) -> '␍' // SYMBOL FOR CARRIAGE RETURN + // + // Also encode invalid UTF-8 bytes as json doesn't handle them properly. + // + // https://www.opendrive.com/wp-content/uploads/guides/OpenDrive_API_guide.pdf + Default: (encoder.Base | + encoder.EncodeWin | + encoder.EncodeLeftCrLfHtVt | + encoder.EncodeRightCrLfHtVt | + encoder.EncodeBackSlash | + encoder.EncodeLeftSpace | + encoder.EncodeRightSpace | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index 5cb4fda9d..08c0e97d8 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -26,7 +26,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/dircache" @@ -84,7 +83,12 @@ func init() { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Pcloud, + // Encode invalid UTF-8 bytes as json doesn't handle them properly. + // + // TODO: Investigate Unicode simplification (＼ gets converted to \ server-side) + Default: (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/premiumizeme/premiumizeme.go b/backend/premiumizeme/premiumizeme.go index 608d88eec..846a35a4b 100644 --- a/backend/premiumizeme/premiumizeme.go +++ b/backend/premiumizeme/premiumizeme.go @@ -35,7 +35,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -97,7 +96,11 @@ This is not normally used - use oauth instead. Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.PremiumizeMe, + // Encode invalid UTF-8 bytes as json doesn't handle them properly. + Default: (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeDoubleQuote | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/putio/putio.go b/backend/putio/putio.go index 52992d062..fcb59cecd 100644 --- a/backend/putio/putio.go +++ b/backend/putio/putio.go @@ -9,7 +9,6 @@ import ( "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/lib/dircache" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/oauthutil" @@ -70,7 +69,12 @@ func init() { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Putio, + // Note that \ is renamed to - + // + // Encode invalid UTF-8 bytes as json doesn't handle them properly. + Default: (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go index 8505e6f39..5bde80eb0 100644 --- a/backend/qingstor/qingstor.go +++ b/backend/qingstor/qingstor.go @@ -21,7 +21,6 @@ import ( "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/walk" @@ -117,7 +116,9 @@ this may help to speed up the transfers.`, Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.QingStor, + Default: (encoder.EncodeInvalidUtf8 | + encoder.EncodeCtl | + encoder.EncodeSlash), }}, }) } diff --git a/backend/s3/s3.go b/backend/s3/s3.go index a3cbde854..6441b0aa1 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -49,7 +49,6 @@ import ( "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -815,7 +814,19 @@ In Ceph, this can be increased with the "rgw list buckets max chunk" option. Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.S3, + // Any UTF-8 character is valid in a key, however it can't handle + // invalid UTF-8 and / have a special meaning. + // + // The SDK can't seem to handle uploading files called '.' + // + // FIXME would be nice to add + // - initial / encoding + // - doubled / encoding + // - trailing / encoding + // so that AWS keys are always valid file names + Default: (encoder.EncodeInvalidUtf8 | + encoder.EncodeSlash | + encoder.EncodeDot), }}, }) } diff --git a/backend/sharefile/sharefile.go b/backend/sharefile/sharefile.go index 52945f2e7..2be175003 100644 --- a/backend/sharefile/sharefile.go +++ b/backend/sharefile/sharefile.go @@ -91,7 +91,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/dircache" @@ -208,7 +207,15 @@ be set manually to something like: https://XXX.sharefile.com Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Sharefile, + Default: (encoder.Base | + encoder.EncodeWin | // :?"*<>| + encoder.EncodeBackSlash | // \ + encoder.EncodeCtl | + encoder.EncodeRightSpace | + encoder.EncodeRightPeriod | + encoder.EncodeLeftSpace | + encoder.EncodeLeftPeriod | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/backend/swift/swift.go b/backend/swift/swift.go index d7027d7ec..108e597fa 100644 --- a/backend/swift/swift.go +++ b/backend/swift/swift.go @@ -19,7 +19,6 @@ import ( "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -66,7 +65,8 @@ copy operations.`, Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Swift, + Default: (encoder.EncodeInvalidUtf8 | + encoder.EncodeSlash), }} // Register with Fs diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.go index f477cf8c4..4fb9eb0b2 100644 --- a/backend/yandex/yandex.go +++ b/backend/yandex/yandex.go @@ -20,7 +20,6 @@ import ( "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/encoder" @@ -83,7 +82,10 @@ func init() { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, - Default: encodings.Yandex, + // Of the control characters \t \n \r are allowed + // it doesn't seem worth making an exception for this + Default: (encoder.Display | + encoder.EncodeInvalidUtf8), }}, }) } diff --git a/fs/encodings/encodings.go b/fs/encodings/encodings.go deleted file mode 100644 index b4d193528..000000000 --- a/fs/encodings/encodings.go +++ /dev/null @@ -1,419 +0,0 @@ -// +build !noencode - -package encodings - -import ( - "runtime" - "strings" - - "github.com/rclone/rclone/lib/encoder" -) - -// LocalUnix is the encoding used by the local backend for non windows platforms -const LocalUnix = encoder.Base - -// LocalMacOS is the encoding used by the local backend for macOS -// -// macOS can't store invalid UTF-8, it converts them into %XX encoding -const LocalMacOS = (encoder.Base | - encoder.EncodeInvalidUtf8) - -// LocalWindows is the encoding used by the local backend for windows platforms -// -// List of replaced characters: -// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN -// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN -// : (colon) -> '：' // FULLWIDTH COLON -// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK -// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS -// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE -// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK -// * (asterisk) -> '＊' // FULLWIDTH ASTERISK -// -// Additionally names can't end with a period (.) or space ( ). -// List of replaced characters: -// . (period) -> '．' // FULLWIDTH FULL STOP -// (space) -> '␠' // SYMBOL FOR SPACE -// -// Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16. -// -// https://docs.microsoft.com/de-de/windows/desktop/FileIO/naming-a-file#naming-conventions -const LocalWindows = (encoder.Base | - encoder.EncodeWin | - encoder.EncodeBackSlash | - encoder.EncodeCtl | - encoder.EncodeRightSpace | - encoder.EncodeRightPeriod | - encoder.EncodeInvalidUtf8) - -// AmazonCloudDrive is the encoding used by the amazonclouddrive backend -// -// Encode invalid UTF-8 bytes as json doesn't handle them properly. -const AmazonCloudDrive = (encoder.Base | - encoder.EncodeInvalidUtf8) - -// B2 is the encoding used by the b2 backend -// -// See: https://www.backblaze.com/b2/docs/files.html -// Encode invalid UTF-8 bytes as json doesn't handle them properly. -// FIXME: allow /, but not leading, trailing or double -const B2 = (encoder.Display | - encoder.EncodeBackSlash | - encoder.EncodeInvalidUtf8) - -// Box is the encoding used by the box backend -// -// From https://developer.box.com/docs/error-codes#section-400-bad-request : -// > Box only supports file or folder names that are 255 characters or less. -// > File names containing non-printable ascii, "/" or "\", names with leading -// > or trailing spaces, and the special names “.” and “..” are also unsupported. -// -// Testing revealed names with leading spaces work fine. -// Also encode invalid UTF-8 bytes as json doesn't handle them properly. -const Box = (encoder.Display | - encoder.EncodeBackSlash | - encoder.EncodeRightSpace | - encoder.EncodeInvalidUtf8) - -// Drive is the encoding used by the drive backend -// -// Encode invalid UTF-8 bytes as json doesn't handle them properly. -// Don't encode / as it's a valid name character in drive. -const Drive = (encoder.EncodeInvalidUtf8) - -// Dropbox is the encoding used by the dropbox backend -// -// https://www.dropbox.com/help/syncing-uploads/files-not-syncing lists / and \ -// as invalid characters. -// Testing revealed names with trailing spaces and the DEL character don't work. -// Also encode invalid UTF-8 bytes as json doesn't handle them properly. -const Dropbox = (encoder.Base | - encoder.EncodeBackSlash | - encoder.EncodeDel | - encoder.EncodeRightSpace | - encoder.EncodeInvalidUtf8) - -// GoogleCloudStorage is the encoding used by the googlecloudstorage backend -const GoogleCloudStorage = (encoder.Base | - encoder.EncodeCrLf | - encoder.EncodeInvalidUtf8) - -// JottaCloud is the encoding used by the jottacloud backend -// -// Encode invalid UTF-8 bytes as xml doesn't handle them properly. -// -// Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|' -const JottaCloud = (encoder.Display | - encoder.EncodeWin | // :?"*<>| - encoder.EncodeInvalidUtf8) - -// Koofr is the encoding used by the koofr backend -// -// Encode invalid UTF-8 bytes as json doesn't handle them properly. -const Koofr = (encoder.Display | - encoder.EncodeBackSlash | - encoder.EncodeInvalidUtf8) - -// Mailru is the encoding used by the mailru backend -// -// Encode invalid UTF-8 bytes as json doesn't handle them properly. -const Mailru = (encoder.Display | - encoder.EncodeWin | // :?"*<>| - encoder.EncodeBackSlash | - encoder.EncodeInvalidUtf8) - -// Mega is the encoding used by the mega backend -// -// Encode invalid UTF-8 bytes as json doesn't handle them properly. -const Mega = (encoder.Base | - encoder.EncodeInvalidUtf8) - -// OneDrive is the encoding used by the onedrive backend -// -// List of replaced characters: -// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN -// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN -// : (colon) -> '：' // FULLWIDTH COLON -// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK -// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS -// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE -// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK -// * (asterisk) -> '＊' // FULLWIDTH ASTERISK -// # (number sign) -> '＃' // FULLWIDTH NUMBER SIGN -// % (percent sign) -> '％' // FULLWIDTH PERCENT SIGN -// -// Folder names cannot begin with a tilde ('~') -// List of replaced characters: -// ~ (tilde) -> '～' // FULLWIDTH TILDE -// -// Additionally names can't begin with a space ( ) or end with a period (.) or space ( ). -// List of replaced characters: -// . (period) -> '．' // FULLWIDTH FULL STOP -// (space) -> '␠' // SYMBOL FOR SPACE -// -// Also encode invalid UTF-8 bytes as json doesn't handle them. -// -// The OneDrive API documentation lists the set of reserved characters, but -// testing showed this list is incomplete. This are the differences: -// - " (double quote) is rejected, but missing in the documentation -// - space at the end of file and folder names is rejected, but missing in the documentation -// - period at the end of file names is rejected, but missing in the documentation -// -// Adding these restrictions to the OneDrive API documentation yields exactly -// the same rules as the Windows naming conventions. -// -// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online#path-encoding -const OneDrive = (encoder.Display | - encoder.EncodeBackSlash | - encoder.EncodeHashPercent | - encoder.EncodeLeftSpace | - encoder.EncodeLeftTilde | - encoder.EncodeRightPeriod | - encoder.EncodeRightSpace | - encoder.EncodeWin | - encoder.EncodeInvalidUtf8) - -// OpenDrive is the encoding used by the opendrive backend -// -// List of replaced characters: -// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN -// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN -// : (colon) -> '：' // FULLWIDTH COLON -// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK -// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS -// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE -// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK -// * (asterisk) -> '＊' // FULLWIDTH ASTERISK -// -// Additionally names can't begin or end with a ASCII whitespace. -// List of replaced characters: -// (space) -> '␠' // SYMBOL FOR SPACE -// (horizontal tab) -> '␉' // SYMBOL FOR HORIZONTAL TABULATION -// (line feed) -> '␊' // SYMBOL FOR LINE FEED -// (vertical tab) -> '␋' // SYMBOL FOR VERTICAL TABULATION -// (carriage return) -> '␍' // SYMBOL FOR CARRIAGE RETURN -// -// Also encode invalid UTF-8 bytes as json doesn't handle them properly. -// -// https://www.opendrive.com/wp-content/uploads/guides/OpenDrive_API_guide.pdf -const OpenDrive = (encoder.Base | - encoder.EncodeWin | - encoder.EncodeLeftCrLfHtVt | - encoder.EncodeRightCrLfHtVt | - encoder.EncodeBackSlash | - encoder.EncodeLeftSpace | - encoder.EncodeRightSpace | - encoder.EncodeInvalidUtf8) - -// PremiumizeMe is the encoding used by the premiumizeme backend -// -// Encode invalid UTF-8 bytes as json doesn't handle them properly. -const PremiumizeMe = (encoder.Display | - encoder.EncodeBackSlash | - encoder.EncodeDoubleQuote | - encoder.EncodeInvalidUtf8) - -// Pcloud is the encoding used by the pcloud backend -// -// Encode invalid UTF-8 bytes as json doesn't handle them properly. -// -// TODO: Investigate Unicode simplification (＼ gets converted to \ server-side) -const Pcloud = (encoder.Display | - encoder.EncodeBackSlash | - encoder.EncodeInvalidUtf8) - -// Putio is the encoding used by the putio backend -// -// Note that \ is renamed to - -// -// Encode invalid UTF-8 bytes as json doesn't handle them properly. -const Putio = (encoder.Display | - encoder.EncodeBackSlash | - encoder.EncodeInvalidUtf8) - -// Fichier is the encoding used by the fichier backend -// -// Characters that need escaping -// -// '\\': '＼', // FULLWIDTH REVERSE SOLIDUS -// '<': '＜', // FULLWIDTH LESS-THAN SIGN -// '>': '＞', // FULLWIDTH GREATER-THAN SIGN -// '"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved -// '\'': '＇', // FULLWIDTH APOSTROPHE -// '$': '＄', // FULLWIDTH DOLLAR SIGN -// '`': '｀', // FULLWIDTH GRAVE ACCENT -// -// Leading space and trailing space -const Fichier = (encoder.Display | - encoder.EncodeBackSlash | - encoder.EncodeSingleQuote | - encoder.EncodeBackQuote | - encoder.EncodeDoubleQuote | - encoder.EncodeLtGt | - encoder.EncodeDollar | - encoder.EncodeLeftSpace | - encoder.EncodeRightSpace | - encoder.EncodeInvalidUtf8) - -// FTP is the encoding used by the ftp backend -// -// The FTP protocal can't handle trailing spaces (for instance -// pureftpd turns them into _) -// -// proftpd can't handle '*' in file names -// pureftpd can't handle '[', ']' or '*' -const FTP = (encoder.Display | - encoder.EncodeRightSpace) - -// S3 is the encoding used by the s3 backend -// -// Any UTF-8 character is valid in a key, however it can't handle -// invalid UTF-8 and / have a special meaning. -// -// The SDK can't seem to handle uploading files called '.' -// -// FIXME would be nice to add -// - initial / encoding -// - doubled / encoding -// - trailing / encoding -// so that AWS keys are always valid file names -const S3 = (encoder.EncodeInvalidUtf8 | - encoder.EncodeSlash | - encoder.EncodeDot) - -// Swift is the encoding used by the swift backend -const Swift = (encoder.EncodeInvalidUtf8 | - encoder.EncodeSlash) - -// AzureBlob is the encoding used by the azureblob backend -const AzureBlob = (encoder.EncodeInvalidUtf8 | - encoder.EncodeSlash | - encoder.EncodeCtl | - encoder.EncodeDel | - encoder.EncodeBackSlash | - encoder.EncodeRightPeriod) - -// QingStor is the encoding used by the qingstor backend -const QingStor = (encoder.EncodeInvalidUtf8 | - encoder.EncodeCtl | - encoder.EncodeSlash) - -// Sharefile is the encoding used by the sharefile backend -const Sharefile = (encoder.Base | - encoder.EncodeWin | // :?"*<>| - encoder.EncodeBackSlash | // \ - encoder.EncodeCtl | - encoder.EncodeRightSpace | - encoder.EncodeRightPeriod | - encoder.EncodeLeftSpace | - encoder.EncodeLeftPeriod | - encoder.EncodeInvalidUtf8) - -// Yandex is the encoding used by the yandex backend -// -// Of the control characters \t \n \r are allowed -// it doesn't seem worth making an exception for this -const Yandex = (encoder.Display | - encoder.EncodeInvalidUtf8) - -// ByName returns the encoder for a give backend name or nil -func ByName(name string) encoder.Encoder { - switch strings.ToLower(name) { - case "base": - return encoder.Base - case "display": - return encoder.Display - case "amazonclouddrive": - return AmazonCloudDrive - case "azureblob": - return AzureBlob - case "b2": - return B2 - case "box": - return Box - //case "cache": - case "drive": - return Drive - case "dropbox": - return Dropbox - //case "ftp": - case "ficher": - return Fichier - case "googlecloudstorage": - return GoogleCloudStorage - //case "http": - //case "hubic": - case "jottacloud": - return JottaCloud - case "koofr": - return Koofr - case "local": - return Local() - case "local-windows", "windows": - return LocalWindows - case "local-unix", "unix": - return LocalUnix - case "local-macos", "macos": - return LocalMacOS - case "mailru": - return Mailru - case "mega": - return Mega - case "onedrive": - return OneDrive - case "opendrive": - return OpenDrive - case "premiumizeme": - return PremiumizeMe - case "pcloud": - return Pcloud - case "qingstor": - return QingStor - case "s3": - return S3 - case "sharefile": - return Sharefile - //case "sftp": - case "swift": - return Swift - //case "webdav": - case "yandex": - return Yandex - default: - return nil - } -} - -// Local returns the local encoding for the current platform -func Local() encoder.MultiEncoder { - if runtime.GOOS == "windows" { - return LocalWindows - } - return LocalUnix -} - -// Names returns the list of known encodings as accepted by ByName -func Names() []string { - return []string{ - "base", - "display", - "amazonclouddrive", - "b2", - "box", - "drive", - "dropbox", - "googlecloudstorage", - "jottacloud", - "koofr", - "local-unix", - "local-windows", - "local-macos", - "local", - "mega", - "onedrive", - "opendrive", - "premiumizeme", - "pcloud", - "sharefile", - } -} diff --git a/fs/encodings/encodings_noencode.go b/fs/encodings/encodings_noencode.go deleted file mode 100644 index 932c86aca..000000000 --- a/fs/encodings/encodings_noencode.go +++ /dev/null @@ -1,57 +0,0 @@ -// +build noencode - -package encodings - -import ( - "github.com/rclone/rclone/lib/encoder" -) - -// Fake encodings used for testing -const ( - Base = encoder.MultiEncoder( - encoder.EncodeZero | - encoder.EncodeSlash) - Display = Base - LocalUnix = Base - LocalWindows = Base - AmazonCloudDrive = Base - AzureBlob = Base - B2 = Base - Box = Base - Drive = Base - Dropbox = Base - FTP = Base - Fichier = Base - GoogleCloudStorage = Base - JottaCloud = Base - Koofr = Base - Mailru = Base - Mega = Base - OneDrive = Base - OpenDrive = Base - Pcloud = Base - PremiumizeMe = Base - Putio = Base - QingStor = Base - S3 = Base - Sharefile = Base - Swift = Base - Yandex = Base -) - -// ByName returns the encoder for a give backend name or nil -func ByName(name string) encoder.Encoder { - return Base -} - -// Local returns the local encoding for the current platform -func Local() encoder.MultiEncoder { - return Base -} - -// Names returns the list of known encodings as accepted by ByName -func Names() []string { - return []string{ - "base", - } -}