extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-07 17:14:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

onedrive: shared folder support - fixes #1200

Browse Source
This commit is contained in:
Yoni Jah 2018-07-12 01:48:59 +08:00 committed by Nick Craig-Wood
parent 793f594b07
commit b1bd17a220
2 changed files with 256 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
backend/onedrive/api/types.go
View File

@ -2,7 +2,10 @@
package api
import "time"
import (
"strings"
"time"
)
const (
timeFormat = `"` + time.RFC3339 + `"`
@ -93,6 +96,22 @@ type ItemReference struct {
Path string `json:"path"` // Path that used to navigate to the item. Read/Write.
}
// RemoteItemFacet groups data needed to reference a OneDrive remote item
type RemoteItemFacet struct {
ID string `json:"id"` // The unique identifier of the item within the remote Drive. Read-only.
Name string `json:"name"` // The name of the item (filename and extension). Read-write.
CreatedBy IdentitySet `json:"createdBy"` // Identity of the user, device, and application which created the item. Read-only.
LastModifiedBy IdentitySet `json:"lastModifiedBy"` // Identity of the user, device, and application which last modified the item. Read-only.
CreatedDateTime Timestamp `json:"createdDateTime"` // Date and time of item creation. Read-only.
LastModifiedDateTime Timestamp `json:"lastModifiedDateTime"` // Date and time the item was last modified. Read-only.
Folder *FolderFacet `json:"folder"` // Folder metadata, if the item is a folder. Read-only.
File *FileFacet `json:"file"` // File metadata, if the item is a file. Read-only.
FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write.
ParentReference *ItemReference `json:"parentReference"` // Parent information, if the item has a parent. Read-write.
Size int64 `json:"size"` // Size of the item in bytes. Read-only.
WebURL string `json:"webUrl"` // URL that displays the resource in the browser. Read-only.
}
// FolderFacet groups folder-related data on OneDrive into a single structure
type FolderFacet struct {
ChildCount int64 `json:"childCount"` // Number of children contained immediately within this container.
@ -143,6 +162,7 @@ type Item struct {
Description string `json:"description"` // Provide a user-visible description of the item. Read-write.
Folder *FolderFacet `json:"folder"` // Folder metadata, if the item is a folder. Read-only.
File *FileFacet `json:"file"` // File metadata, if the item is a file. Read-only.
RemoteItem *RemoteItemFacet `json:"remoteItem"` // Remote Item metadata, if the item is a remote shared item. Read-only.
FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write.
// Image *ImageFacet `json:"image"` // Image metadata, if the item is an image. Read-only.
// Photo *PhotoFacet `json:"photo"` // Photo metadata, if the item is a photo. Read-only.
@ -228,3 +248,112 @@ type AsyncOperationStatus struct {
PercentageComplete float64 `json:"percentageComplete"` // An float value between 0 and 100 that indicates the percentage complete.
Status string `json:"status"` // A string value that maps to an enumeration of possible values about the status of the job. "notStarted | inProgress | completed | updating | failed | deletePending | deleteFailed | waiting"
}
// GetID returns a normalized ID of the item
// If DriveID is known it will be prefixed to the ID with # seperator
func (i *Item) GetID() string {
if i.IsRemote() && i.RemoteItem.ID != "" {
return i.RemoteItem.ParentReference.DriveID + "#" + i.RemoteItem.ID
} else if i.ParentReference != nil && strings.Index(i.ID, "#") == -1 {
return i.ParentReference.DriveID + "#" + i.ID
}
return i.ID
}
// GetDriveID returns a normalized ParentReferance of the item
func (i *Item) GetDriveID() string {
return i.GetParentReferance().DriveID
}
// GetName returns a normalized Name of the item
func (i *Item) GetName() string {
if i.IsRemote() && i.RemoteItem.Name != "" {
return i.RemoteItem.Name
}
return i.Name
}
// GetFolder returns a normalized Folder of the item
func (i *Item) GetFolder() *FolderFacet {
if i.IsRemote() && i.RemoteItem.Folder != nil {
return i.RemoteItem.Folder
}
return i.Folder
}
// GetFile returns a normalized File of the item
func (i *Item) GetFile() *FileFacet {
if i.IsRemote() && i.RemoteItem.File != nil {
return i.RemoteItem.File
}
return i.File
}
// GetFileSystemInfo returns a normalized FileSystemInfo of the item
func (i *Item) GetFileSystemInfo() *FileSystemInfoFacet {
if i.IsRemote() && i.RemoteItem.FileSystemInfo != nil {
return i.RemoteItem.FileSystemInfo
}
return i.FileSystemInfo
}
// GetSize returns a normalized Size of the item
func (i *Item) GetSize() int64 {
if i.IsRemote() && i.RemoteItem.Size != 0 {
return i.RemoteItem.Size
}
return i.Size
}
// GetWebURL returns a normalized WebURL of the item
func (i *Item) GetWebURL() string {
if i.IsRemote() && i.RemoteItem.WebURL != "" {
return i.RemoteItem.WebURL
}
return i.WebURL
}
// GetCreatedBy returns a normalized CreatedBy of the item
func (i *Item) GetCreatedBy() IdentitySet {
if i.IsRemote() && i.RemoteItem.CreatedBy != (IdentitySet{}) {
return i.RemoteItem.CreatedBy
}
return i.CreatedBy
}
// GetLastModifiedBy returns a normalized LastModifiedBy of the item
func (i *Item) GetLastModifiedBy() IdentitySet {
if i.IsRemote() && i.RemoteItem.LastModifiedBy != (IdentitySet{}) {
return i.RemoteItem.LastModifiedBy
}
return i.LastModifiedBy
}
// GetCreatedDateTime returns a normalized CreatedDateTime of the item
func (i *Item) GetCreatedDateTime() Timestamp {
if i.IsRemote() && i.RemoteItem.CreatedDateTime != (Timestamp{}) {
return i.RemoteItem.CreatedDateTime
}
return i.CreatedDateTime
}
// GetLastModifiedDateTime returns a normalized LastModifiedDateTime of the item
func (i *Item) GetLastModifiedDateTime() Timestamp {
if i.IsRemote() && i.RemoteItem.LastModifiedDateTime != (Timestamp{}) {
return i.RemoteItem.LastModifiedDateTime
}
return i.LastModifiedDateTime
}
// GetParentReferance returns a normalized ParentReferance of the item
func (i *Item) GetParentReferance() *ItemReference {
if i.IsRemote() && i.ParentReference == nil {
return i.RemoteItem.ParentReference
}
return i.ParentReference
}
// IsRemote checks if item is a remote item
func (i *Item) IsRemote() bool {
return i.RemoteItem != nil
}

193
backend/onedrive/onedrive.go
View File

@ -75,6 +75,7 @@ var (
oauthBusinessResource = oauth2.SetAuthURLParam("resource", discoveryServiceURL)
chunkSize = fs.SizeSuffix(10 * 1024 * 1024)
sharedURL = "https://api.onedrive.com/v1.0/drives" // root URL for remote shared resources
)
// Register with Fs
@ -325,6 +326,7 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Respon
resp, err = f.srv.CallJSON(&opts, nil, &info)
return shouldRetry(resp, err)
})
return info, resp, err
}
@ -357,6 +359,7 @@ func NewFs(name, root string) (fs.Fs, error) {
// business account setup
oauthConfig = oauthBusinessConfig
rootURL = resourceURL + "_api/v2.0/drives/me"
sharedURL = resourceURL + "_api/v2.0/drives"
// update the URL in the AuthOptions
oauthBusinessResource = oauth2.SetAuthURLParam("resource", resourceURL)
@ -482,21 +485,18 @@ func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err er
}
return "", false, err
}
if info.Folder == nil {
if info.GetFolder() == nil {
return "", false, errors.New("found file when looking for folder")
}
return info.ID, true, nil
return info.GetID(), true, nil
}
// CreateDir makes a directory with pathID as parent and name leaf
func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) {
// fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf)
func (f *Fs) CreateDir(dirID, leaf string) (newID string, err error) {
// fs.Debugf(f, "CreateDir(%q, %q)\n", dirID, leaf)
var resp *http.Response
var info *api.Item
opts := rest.Opts{
Method: "POST",
Path: "/items/" + pathID + "/children",
}
opts := newOptsCall(dirID, "POST", "/children")
mkdir := api.CreateItemRequest{
Name: replaceReservedChars(leaf),
ConflictBehavior: "fail",
@ -509,8 +509,9 @@ func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) {
//fmt.Printf("...Error %v\n", err)
return "", err
}
//fmt.Printf("...Id %q\n", *info.Id)
return info.ID, nil
return info.GetID(), nil
}
// list the objects into the function supplied
@ -527,10 +528,8 @@ type listAllFn func(*api.Item) bool
func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
// Top parameter asks for bigger pages of data
// https://dev.onedrive.com/odata/optional-query-parameters.htm
opts := rest.Opts{
Method: "GET",
Path: "/items/" + dirID + "/children?top=1000",
}
opts := newOptsCall(dirID, "GET", "/children?top=1000")
OUTER:
for {
var result api.ListChildrenResponse
@ -547,7 +546,7 @@ OUTER:
}
for i := range result.Value {
item := &result.Value[i]
isFolder := item.Folder != nil
isFolder := item.GetFolder() != nil
if isFolder {
if filesOnly {
continue
@ -560,7 +559,7 @@ OUTER:
if item.Deleted != nil {
continue
}
item.Name = restoreReservedChars(item.Name)
item.Name = restoreReservedChars(item.GetName())
if fn(item) {
found = true
break OUTER
@ -595,13 +594,15 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
}
var iErr error
_, err = f.listAll(directoryID, false, false, func(info *api.Item) bool {
remote := path.Join(dir, info.Name)
if info.Folder != nil {
remote := path.Join(dir, info.GetName())
folder := info.GetFolder()
if folder != nil {
// cache the directory ID for later lookups
f.dirCache.Put(remote, info.ID)
d := fs.NewDir(remote, time.Time(info.LastModifiedDateTime)).SetID(info.ID)
if info.Folder != nil {
d.SetItems(info.Folder.ChildCount)
id := info.GetID()
f.dirCache.Put(remote, id)
d := fs.NewDir(remote, time.Time(info.GetLastModifiedDateTime())).SetID(id)
if folder != nil {
d.SetItems(folder.ChildCount)
}
entries = append(entries, d)
} else {
@ -674,11 +675,9 @@ func (f *Fs) Mkdir(dir string) error {
// deleteObject removes an object by ID
func (f *Fs) deleteObject(id string) error {
opts := rest.Opts{
Method: "DELETE",
Path: "/items/" + id,
NoResponse: true,
}
opts := newOptsCall(id, "DELETE", "")
opts.NoResponse = true
return f.pacer.Call(func() (bool, error) {
resp, err := f.srv.Call(&opts)
return shouldRetry(resp, err)
@ -814,17 +813,17 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
}
// Copy the object
opts := rest.Opts{
Method: "POST",
Path: "/items/" + srcObj.id + "/action.copy",
ExtraHeaders: map[string]string{"Prefer": "respond-async"},
NoResponse: true,
}
opts := newOptsCall(srcObj.id, "POST", "/action.copy")
opts.ExtraHeaders = map[string]string{"Prefer": "respond-async"}
opts.NoResponse = true
id, _, _ := parseDirID(directoryID)
replacedLeaf := replaceReservedChars(leaf)
copy := api.CopyItemRequest{
Name: &replacedLeaf,
ParentReference: api.ItemReference{
ID: directoryID,
ID: id,
},
}
var resp *http.Response
@ -891,14 +890,14 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
}
// Move the object
opts := rest.Opts{
Method: "PATCH",
Path: "/items/" + srcObj.id,
}
opts := newOptsCall(srcObj.id, "PATCH", "")
id, _, _ := parseDirID(directoryID)
move := api.MoveItemRequest{
Name: replaceReservedChars(leaf),
ParentReference: &api.ItemReference{
ID: directoryID,
ID: id,
},
// We set the mod time too as it gets reset otherwise
FileSystemInfo: &api.FileSystemInfoFacet{
@ -1013,35 +1012,37 @@ func (o *Object) Size() int64 {
// setMetaData sets the metadata from info
func (o *Object) setMetaData(info *api.Item) (err error) {
if info.Folder != nil {
if info.GetFolder() != nil {
return errors.Wrapf(fs.ErrorNotAFile, "%q", o.remote)
}
o.hasMetaData = true
o.size = info.Size
o.size = info.GetSize()
// Docs: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/hashes
//
// We use SHA1 for onedrive personal and QuickXorHash for onedrive for business
if info.File != nil {
o.mimeType = info.File.MimeType
if info.File.Hashes.Sha1Hash != "" {
o.sha1 = strings.ToLower(info.File.Hashes.Sha1Hash)
file := info.GetFile()
if file != nil {
o.mimeType = file.MimeType
if file.Hashes.Sha1Hash != "" {
o.sha1 = strings.ToLower(file.Hashes.Sha1Hash)
}
if info.File.Hashes.QuickXorHash != "" {
h, err := base64.StdEncoding.DecodeString(info.File.Hashes.QuickXorHash)
if file.Hashes.QuickXorHash != "" {
h, err := base64.StdEncoding.DecodeString(file.Hashes.QuickXorHash)
if err != nil {
fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", info.File.Hashes.QuickXorHash, err)
fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", file.Hashes.QuickXorHash, err)
} else {
o.quickxorhash = hex.EncodeToString(h)
}
}
}
if info.FileSystemInfo != nil {
o.modTime = time.Time(info.FileSystemInfo.LastModifiedDateTime)
fileSystemInfo := info.GetFileSystemInfo()
if fileSystemInfo != nil {
o.modTime = time.Time(fileSystemInfo.LastModifiedDateTime)
} else {
o.modTime = time.Time(info.LastModifiedDateTime)
o.modTime = time.Time(info.GetLastModifiedDateTime())
}
o.id = info.ID
o.id = info.GetID()
return nil
}
@ -1080,9 +1081,20 @@ func (o *Object) ModTime() time.Time {
// setModTime sets the modification time of the local fs object
func (o *Object) setModTime(modTime time.Time) (*api.Item, error) {
opts := rest.Opts{
Method: "PATCH",
Path: "/root:/" + rest.URLPathEscape(o.srvPath()),
var opts rest.Opts
_, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
_, drive, rootURL := parseDirID(directoryID)
if drive != "" {
opts = rest.Opts{
Method: "PATCH",
RootURL: rootURL,
Path: "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()),
}
} else {
opts = rest.Opts{
Method: "PATCH",
Path: "/root:/" + rest.URLPathEscape(o.srvPath()),
}
}
update := api.SetFileSystemInfo{
FileSystemInfo: api.FileSystemInfoFacet{
@ -1119,11 +1131,9 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
}
fs.FixRangeOption(options, o.size)
var resp *http.Response
opts := rest.Opts{
Method: "GET",
Path: "/items/" + o.id + "/content",
Options: options,
}
opts := newOptsCall(o.id, "GET", "/content")
opts.Options = options
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(&opts)
return shouldRetry(resp, err)
@ -1141,9 +1151,20 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
// createUploadSession creates an upload session for the object
func (o *Object) createUploadSession(modTime time.Time) (response *api.CreateUploadResponse, err error) {
opts := rest.Opts{
Method: "POST",
Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/upload.createSession",
leaf, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
id, drive, rootURL := parseDirID(directoryID)
var opts rest.Opts
if drive != "" {
opts = rest.Opts{
Method: "POST",
RootURL: rootURL,
Path: "/" + drive + "/items/" + id + ":/" + rest.URLPathEscape(leaf) + ":/upload.createSession",
}
} else {
opts = rest.Opts{
Method: "POST",
Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/upload.createSession",
}
}
createRequest := api.CreateUploadRequest{}
createRequest.Item.FileSystemInfo.CreatedDateTime = api.Timestamp(modTime)
@ -1251,11 +1272,24 @@ func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (i
// uploadSinglepart uploads a file as a single part
func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
var resp *http.Response
opts := rest.Opts{
Method: "PUT",
Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content",
ContentLength: &size,
Body: in,
var opts rest.Opts
_, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
_, drive, rootURL := parseDirID(directoryID)
if drive != "" {
opts = rest.Opts{
Method: "PUT",
RootURL: rootURL,
Path: "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content",
ContentLength: &size,
Body: in,
}
} else {
opts = rest.Opts{
Method: "PUT",
Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content",
ContentLength: &size,
Body: in,
}
}
// for go1.8 (see release notes) we must nil the Body if we want a
// "Content-Length: 0" header which onedrive requires for all files.
@ -1269,6 +1303,7 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (
if err != nil {
return nil, err
}
err = o.setMetaData(info)
if err != nil {
return nil, err
@ -1315,6 +1350,30 @@ func (o *Object) ID() string {
return o.id
}
func newOptsCall(id string, method string, route string) (opts rest.Opts) {
id, drive, rootURL := parseDirID(id)
if drive != "" {
return rest.Opts{
Method: method,
RootURL: rootURL,
Path: "/" + drive + "/items/" + id + route,
}
}
return rest.Opts{
Method: method,
Path: "/items/" + id + route,
}
}
func parseDirID(ID string) (string, string, string) {
if strings.Index(ID, "#") >= 0 {
s := strings.Split(ID, "#")
return s[1], s[0], sharedURL
}
return ID, "", ""
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)