extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-02-16 18:41:54 +01:00
Code Issues Packages Projects Releases Wiki Activity

backend/protondrive: improve performance of Proton Drive backend

Browse Source 
This change removes redundant calls to the Proton Drive Bridge when
creating Objects. Specifically, the function List() would get a
directory listing, get a link for each file, construct a remote path
from that link, then get a link for that remote path again by calling
getObjectLink() unnecessarily. This change removes that unnecessary
call, and tidies up a couple of functions around this with unused
parameters.

Related to performance issues reported in #7322 and #7413
This commit is contained in:
Lawrence Murray 2024-09-19 00:15:24 +07:00 committed by GitHub
parent 1a9e6a527d
commit c669f4e218
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 22 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
backend/protondrive/protondrive.go
View File

@ -449,7 +449,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// No root so return old f // No root so return old f
return f, nil return f, nil
} }
_, err := tempF.newObjectWithLink(ctx, remote, nil) _, err := tempF.newObject(ctx, remote)
if err != nil { if err != nil {
if err == fs.ErrorObjectNotFound { if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f // File doesn't exist so return old f
@ -487,7 +487,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
// ErrorIsDir if possible without doing any extra work, // ErrorIsDir if possible without doing any extra work,
// otherwise ErrorObjectNotFound. // otherwise ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return f.newObjectWithLink(ctx, remote, nil) return f.newObject(ctx, remote)
} }
func (f *Fs) getObjectLink(ctx context.Context, remote string) (*proton.Link, error) { func (f *Fs) getObjectLink(ctx context.Context, remote string) (*proton.Link, error) {
@ -516,35 +516,27 @@ func (f *Fs) getObjectLink(ctx context.Context, remote string) (*proton.Link, er
return link, nil return link, nil
} }
// readMetaDataForRemote reads the metadata from the remote // readMetaDataForLink reads the metadata from the remote
func (f *Fs) readMetaDataForRemote(ctx context.Context, remote string, _link *proton.Link) (*proton.Link, *protonDriveAPI.FileSystemAttrs, error) { func (f *Fs) readMetaDataForLink(ctx context.Context, link *proton.Link) (*protonDriveAPI.FileSystemAttrs, error) {
link, err := f.getObjectLink(ctx, remote)
if err != nil {
return nil, nil, err
}
var fileSystemAttrs *protonDriveAPI.FileSystemAttrs var fileSystemAttrs *protonDriveAPI.FileSystemAttrs
var err error
if err = f.pacer.Call(func() (bool, error) { if err = f.pacer.Call(func() (bool, error) {
fileSystemAttrs, err = f.protonDrive.GetActiveRevisionAttrs(ctx, link) fileSystemAttrs, err = f.protonDrive.GetActiveRevisionAttrs(ctx, link)
return shouldRetry(ctx, err) return shouldRetry(ctx, err)
}); err != nil { }); err != nil {
return nil, nil, err return nil, err
} }
return link, fileSystemAttrs, nil return fileSystemAttrs, nil
} }
// readMetaData gets the metadata if it hasn't already been fetched // Return an Object from a path and link
// //
// it also sets the info // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (o *Object) readMetaData(ctx context.Context, link *proton.Link) (err error) { func (f *Fs) newObjectWithLink(ctx context.Context, remote string, link *proton.Link) (fs.Object, error) {
if o.link != nil { o := &Object{
return nil fs: f,
} remote: remote,
link, fileSystemAttrs, err := o.fs.readMetaDataForRemote(ctx, o.remote, link)
if err != nil {
return err
} }
o.id = link.LinkID o.id = link.LinkID
@ -554,6 +546,10 @@ func (o *Object) readMetaData(ctx context.Context, link *proton.Link) (err error
o.mimetype = link.MIMEType o.mimetype = link.MIMEType
o.link = link o.link = link
fileSystemAttrs, err := o.fs.readMetaDataForLink(ctx, link)
if err != nil {
return nil, err
}
if fileSystemAttrs != nil { if fileSystemAttrs != nil {
o.modTime = fileSystemAttrs.ModificationTime o.modTime = fileSystemAttrs.ModificationTime
o.originalSize = &fileSystemAttrs.Size o.originalSize = &fileSystemAttrs.Size
@ -561,23 +557,18 @@ func (o *Object) readMetaData(ctx context.Context, link *proton.Link) (err error
o.digests = &fileSystemAttrs.Digests o.digests = &fileSystemAttrs.Digests
} }
return nil return o, nil
} }
// Return an Object from a path // Return an Object from a path only
// //
// If it can't be found it returns the error fs.ErrorObjectNotFound. // If it can't be found it returns the error fs.ErrorObjectNotFound.
func (f *Fs) newObjectWithLink(ctx context.Context, remote string, link *proton.Link) (fs.Object, error) { func (f *Fs) newObject(ctx context.Context, remote string) (fs.Object, error) {
o := &Object{ link, err := f.getObjectLink(ctx, remote)
fs: f,
remote: remote,
}
err := o.readMetaData(ctx, link)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return o, nil return f.newObjectWithLink(ctx, remote, link)
} }
// List the objects and directories in dir into entries. The // List the objects and directories in dir into entries. The