diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 054b8ed6a..fd2590a8a 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -28,6 +28,7 @@ import ( "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/encodings" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" @@ -60,6 +61,8 @@ const ( emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1" ) +const enc = encodings.AzureBlob + // Register with Fs func init() { fs.Register(&fs.RegInfo{ @@ -208,7 +211,8 @@ func parsePath(path string) (root string) { // split returns container and containerPath from the rootRelativePath // relative to f.root func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) { - return bucket.Split(path.Join(f.root, rootRelativePath)) + containerName, containerPath = bucket.Split(path.Join(f.root, rootRelativePath)) + return enc.FromStandardName(containerName), enc.FromStandardPath(containerPath) } // split returns container and containerPath from the object @@ -575,18 +579,18 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC } // Advance marker to next marker = response.NextMarker - for i := range response.Segment.BlobItems { file := &response.Segment.BlobItems[i] // Finish if file name no longer has prefix // if prefix != "" && !strings.HasPrefix(file.Name, prefix) { // return nil // } - if !strings.HasPrefix(file.Name, prefix) { - fs.Debugf(f, "Odd name received %q", file.Name) + remote := enc.ToStandardPath(file.Name) + if !strings.HasPrefix(remote, prefix) { + fs.Debugf(f, "Odd name received %q", remote) continue } - remote := file.Name[len(prefix):] + remote = remote[len(prefix):] if isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote) { continue // skip directory marker } @@ -602,6 +606,7 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC // Send the subdirectories for _, remote := range response.Segment.BlobPrefixes { remote := strings.TrimRight(remote.Name, "/") + remote = enc.ToStandardPath(remote) if !strings.HasPrefix(remote, prefix) { fs.Debugf(f, "Odd directory name received %q", remote) continue @@ -665,7 +670,7 @@ func (f *Fs) listContainers(ctx context.Context) (entries fs.DirEntries, err err return entries, nil } err = f.listContainersToFn(func(container *azblob.ContainerItem) error { - d := fs.NewDir(container.Name, container.Properties.LastModified) + d := fs.NewDir(enc.ToStandardName(container.Name), container.Properties.LastModified) f.cache.MarkOK(container.Name) entries = append(entries, d) return nil diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md index 7dd97a1f8..ba0be75fb 100644 --- a/docs/content/azureblob.md +++ b/docs/content/azureblob.md @@ -81,6 +81,26 @@ key. It is stored using RFC3339 Format time with nanosecond precision. The metadata is supplied during directory listings so there is no overhead to using it. +### Restricted filename characters + +In addition to the [default restricted characters set](/overview/#restricted-characters) +the following characters are also replaced: + +| Character | Value | Replacement | +| --------- |:-----:|:-----------:| +| / | 0x2F | / | +| \ | 0x5C | \ | + +File names can also not end with the following characters. +These only get replaced if they are last character in the name: + +| Character | Value | Replacement | +| --------- |:-----:|:-----------:| +| . | 0x2E | . | + +Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8), +as they can't be used in JSON strings. + ### Hashes ### MD5 hashes are stored with blobs. However blobs that were uploaded in diff --git a/fs/encodings/encodings.go b/fs/encodings/encodings.go index 04bb6c3f5..dcbda8854 100644 --- a/fs/encodings/encodings.go +++ b/fs/encodings/encodings.go @@ -262,6 +262,14 @@ const Swift = encoder.MultiEncoder( encoder.EncodeInvalidUtf8 | encoder.EncodeSlash) +// AzureBlob is the encoding used by the azureblob backend +const AzureBlob = encoder.MultiEncoder( + encoder.EncodeInvalidUtf8 | + encoder.EncodeSlash | + encoder.EncodeCtl | + encoder.EncodeDel | + encoder.EncodeBackSlash | + encoder.EncodeRightPeriod) // ByName returns the encoder for a give backend name or nil func ByName(name string) encoder.Encoder { @@ -272,7 +280,8 @@ func ByName(name string) encoder.Encoder { return Display case "amazonclouddrive": return AmazonCloudDrive - //case "azureblob": + case "azureblob": + return AzureBlob case "b2": return B2 case "box":