From 078d202f39f4a174bb27c6704ec78777f86eefb1 Mon Sep 17 00:00:00 2001 From: yuval-cloudinary <46710068+yuval-cloudinary@users.noreply.github.com> Date: Sun, 23 Feb 2025 13:56:32 +0200 Subject: [PATCH] cloudinary: automatically add/remove known media files extensions #8416 --- backend/cloudinary/api/types.go | 2 +- backend/cloudinary/cloudinary.go | 69 ++++++++++++++++++++++++++------ docs/content/cloudinary.md | 22 ++++++++++ 3 files changed, 79 insertions(+), 14 deletions(-) diff --git a/backend/cloudinary/api/types.go b/backend/cloudinary/api/types.go index 0a635a56b..0c9bc1678 100644 --- a/backend/cloudinary/api/types.go +++ b/backend/cloudinary/api/types.go @@ -18,7 +18,7 @@ type CloudinaryEncoder interface { ToStandardPath(string) string // ToStandardName takes name in this encoding and converts // it in Standard encoding. - ToStandardName(string) string + ToStandardName(string, string) string // Encoded root of the remote (as passed into NewFs) FromStandardFullPath(string) string } diff --git a/backend/cloudinary/cloudinary.go b/backend/cloudinary/cloudinary.go index 6af5bd370..528ff71ea 100644 --- a/backend/cloudinary/cloudinary.go +++ b/backend/cloudinary/cloudinary.go @@ -8,7 +8,9 @@ import ( "fmt" "io" "net/http" + "net/url" "path" + "slices" "strconv" "strings" "time" @@ -103,19 +105,39 @@ func init() { Advanced: true, Help: "Wait N seconds for eventual consistency of the databases that support the backend operation", }, + { + Name: "adjust_media_files_extensions", + Default: true, + Advanced: true, + Help: "Cloudinary handles media formats as a file attribute and strips it from the name, which is unlike most other file systems", + }, + { + Name: "media_extensions", + Default: []string{ + "3ds", "3g2", "3gp", "ai", "arw", "avi", "avif", "bmp", "bw", + "cr2", "cr3", "djvu", "dng", "eps3", "fbx", "flif", "flv", "gif", + "glb", "gltf", "hdp", "heic", "heif", "ico", "indd", "jp2", "jpe", + "jpeg", "jpg", "jxl", "jxr", "m2ts", "mov", "mp4", "mpeg", "mts", + "mxf", "obj", "ogv", "pdf", "ply", "png", "psd", "svg", "tga", + "tif", "tiff", "ts", "u3ma", "usdz", "wdp", "webm", "webp", "wmv"}, + Advanced: true, + Help: "Cloudinary supported media extensions", + }, }, }) } // Options defines the configuration for this backend type Options struct { - CloudName string `config:"cloud_name"` - APIKey string `config:"api_key"` - APISecret string `config:"api_secret"` - UploadPrefix string `config:"upload_prefix"` - UploadPreset string `config:"upload_preset"` - Enc encoder.MultiEncoder `config:"encoding"` - EventuallyConsistentDelay fs.Duration `config:"eventually_consistent_delay"` + CloudName string `config:"cloud_name"` + APIKey string `config:"api_key"` + APISecret string `config:"api_secret"` + UploadPrefix string `config:"upload_prefix"` + UploadPreset string `config:"upload_preset"` + Enc encoder.MultiEncoder `config:"encoding"` + EventuallyConsistentDelay fs.Duration `config:"eventually_consistent_delay"` + MediaExtensions []string `config:"media_extensions"` + AdjustMediaFilesExtensions bool `config:"adjust_media_files_extensions"` } // Fs represents a remote cloudinary server @@ -203,6 +225,18 @@ func (f *Fs) FromStandardPath(s string) string { // FromStandardName implementation of the api.CloudinaryEncoder func (f *Fs) FromStandardName(s string) string { + if f.opt.AdjustMediaFilesExtensions { + parsedURL, err := url.Parse(s) + ext := "" + if err != nil { + fs.Logf(nil, "Error parsing URL: %v", err) + } else { + ext = path.Ext(parsedURL.Path) + if slices.Contains(f.opt.MediaExtensions, strings.ToLower(strings.TrimPrefix(ext, "."))) { + s = strings.TrimSuffix(parsedURL.Path, ext) + } + } + } return strings.ReplaceAll(f.opt.Enc.FromStandardName(s), "&", "\uFF06") } @@ -212,8 +246,20 @@ func (f *Fs) ToStandardPath(s string) string { } // ToStandardName implementation of the api.CloudinaryEncoder -func (f *Fs) ToStandardName(s string) string { - return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&") +func (f *Fs) ToStandardName(s string, assetUrl string) string { + ext := "" + if f.opt.AdjustMediaFilesExtensions { + parsedURL, err := url.Parse(assetUrl) + if err != nil { + fs.Logf(nil, "Error parsing URL: %v", err) + } else { + ext = path.Ext(parsedURL.Path) + if !slices.Contains(f.opt.MediaExtensions, strings.ToLower(strings.TrimPrefix(ext, "."))) { + ext = "" + } + } + } + return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&") + ext } // FromStandardFullPath encodes a full path to Cloudinary standard @@ -331,10 +377,7 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) { } for _, asset := range results.Assets { - remote := api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName) - if dir != "" { - remote = path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName)) - } + remote := path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName, asset.SecureURL)) o := &Object{ fs: f, remote: remote, diff --git a/docs/content/cloudinary.md b/docs/content/cloudinary.md index e4eeb0e3b..c150e7a17 100644 --- a/docs/content/cloudinary.md +++ b/docs/content/cloudinary.md @@ -206,6 +206,28 @@ Properties: - Type: Duration - Default: 0s +#### --cloudinary-adjust-media-files-extensions + +Cloudinary handles media formats as a file attribute and strips it from the name, which is unlike most other file systems + +Properties: + +- Config: adjust_media_files_extensions +- Env Var: RCLONE_CLOUDINARY_ADJUST_MEDIA_FILES_EXTENSIONS +- Type: bool +- Default: true + +#### --cloudinary-media-extensions + +Cloudinary supported media extensions + +Properties: + +- Config: media_extensions +- Env Var: RCLONE_CLOUDINARY_MEDIA_EXTENSIONS +- Type: stringArray +- Default: [3ds 3g2 3gp ai arw avi avif bmp bw cr2 cr3 djvu dng eps3 fbx flif flv gif glb gltf hdp heic heif ico indd jp2 jpe jpeg jpg jxl jxr m2ts mov mp4 mpeg mts mxf obj ogv pdf ply png psd svg tga tif tiff ts u3ma usdz wdp webm webp wmv] + #### --cloudinary-description Description of the remote.