From 51db76fd4754c4861ec14c3b3163f24c5dbaf7a0 Mon Sep 17 00:00:00 2001 From: lostb1t Date: Sat, 28 Sep 2024 08:54:05 +0200 Subject: [PATCH] Add ICloud Drive backend --- README.md | 1 + backend/all/all.go | 1 + backend/iclouddrive/api/client.go | 166 +++ backend/iclouddrive/api/drive.go | 913 +++++++++++++ backend/iclouddrive/api/session.go | 412 ++++++ backend/iclouddrive/iclouddrive.go | 1174 +++++++++++++++++ backend/iclouddrive/iclouddrive_test.go | 18 + .../iclouddrive/iclouddrive_unsupported.go | 7 + bin/make_manual.py | 1 + docs/content/_index.md | 1 + docs/content/docs.md | 1 + docs/content/iclouddrive.md | 156 +++ docs/content/overview.md | 4 +- docs/layouts/chrome/navbar.html | 1 + fstest/test_all/config.yaml | 3 + 15 files changed, 2858 insertions(+), 1 deletion(-) create mode 100644 backend/iclouddrive/api/client.go create mode 100644 backend/iclouddrive/api/drive.go create mode 100644 backend/iclouddrive/api/session.go create mode 100644 backend/iclouddrive/iclouddrive.go create mode 100644 backend/iclouddrive/iclouddrive_test.go create mode 100644 backend/iclouddrive/iclouddrive_unsupported.go create mode 100644 docs/content/iclouddrive.md diff --git a/README.md b/README.md index 9cbd74cb5..866986479 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and * HiDrive [:page_facing_up:](https://rclone.org/hidrive/) * HTTP [:page_facing_up:](https://rclone.org/http/) * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs) + * iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/) * ImageKit [:page_facing_up:](https://rclone.org/imagekit/) * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/) * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/) diff --git a/backend/all/all.go b/backend/all/all.go index a9191c98d..1abd6770f 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -26,6 +26,7 @@ import ( _ "github.com/rclone/rclone/backend/hdfs" _ "github.com/rclone/rclone/backend/hidrive" _ "github.com/rclone/rclone/backend/http" + _ "github.com/rclone/rclone/backend/iclouddrive" _ "github.com/rclone/rclone/backend/imagekit" _ "github.com/rclone/rclone/backend/internetarchive" _ "github.com/rclone/rclone/backend/jottacloud" diff --git a/backend/iclouddrive/api/client.go b/backend/iclouddrive/api/client.go new file mode 100644 index 000000000..7cdf74baf --- /dev/null +++ b/backend/iclouddrive/api/client.go @@ -0,0 +1,166 @@ +// Package api provides functionality for interacting with the iCloud API. +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/lib/rest" +) + +const ( + baseEndpoint = "https://www.icloud.com" + homeEndpoint = "https://www.icloud.com" + setupEndpoint = "https://setup.icloud.com/setup/ws/1" + authEndpoint = "https://idmsa.apple.com/appleauth/auth" +) + +type sessionSave func(*Session) + +// Client defines the client configuration +type Client struct { + appleID string + password string + srv *rest.Client + Session *Session + sessionSaveCallback sessionSave + + drive *DriveService +} + +// New creates a new Client instance with the provided Apple ID, password, trust token, cookies, and session save callback. +// +// Parameters: +// - appleID: the Apple ID of the user. +// - password: the password of the user. +// - trustToken: the trust token for the session. +// - clientID: the client id for the session. +// - cookies: the cookies for the session. +// - sessionSaveCallback: the callback function to save the session. +func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave) (*Client, error) { + icloud := &Client{ + appleID: appleID, + password: password, + srv: rest.NewClient(fshttp.NewClient(context.Background())), + Session: NewSession(), + sessionSaveCallback: sessionSaveCallback, + } + + icloud.Session.TrustToken = trustToken + icloud.Session.Cookies = cookies + icloud.Session.ClientID = clientID + return icloud, nil +} + +// DriveService returns the DriveService instance associated with the Client. +func (c *Client) DriveService() (*DriveService, error) { + var err error + if c.drive == nil { + c.drive, err = NewDriveService(c) + if err != nil { + return nil, err + } + } + return c.drive, nil +} + +// Request makes a request and retries it if the session is invalid. +// +// This function is the main entry point for making requests to the iCloud +// API. If the initial request returns a 401 (Unauthorized), it will try to +// reauthenticate and retry the request. +func (c *Client) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) { + resp, err = c.Session.Request(ctx, opts, request, response) + if err != nil && resp != nil { + // try to reauth + if resp.StatusCode == 401 || resp.StatusCode == 421 { + err = c.Authenticate(ctx) + if err != nil { + return nil, err + } + + if c.Session.Requires2FA() { + return nil, errors.New("trust token expired, please reauth") + } + return c.RequestNoReAuth(ctx, opts, request, response) + } + } + return resp, err +} + +// RequestNoReAuth makes a request without re-authenticating. +// +// This function is useful when you have a session that is already +// authenticated, but you need to make a request without triggering +// a re-authentication. +func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) { + // Make the request without re-authenticating + resp, err = c.Session.Request(ctx, opts, request, response) + return resp, err +} + +// Authenticate authenticates the client with the iCloud API. +func (c *Client) Authenticate(ctx context.Context) error { + if c.Session.Cookies != nil { + if err := c.Session.ValidateSession(ctx); err == nil { + fs.Debugf("icloud", "Valid session, no need to reauth") + return nil + } + c.Session.Cookies = nil + } + + fs.Debugf("icloud", "Authenticating as %s\n", c.appleID) + err := c.Session.SignIn(ctx, c.appleID, c.password) + + if err == nil { + err = c.Session.AuthWithToken(ctx) + if err == nil && c.sessionSaveCallback != nil { + c.sessionSaveCallback(c.Session) + } + } + return err +} + +// SignIn signs in the client using the provided context and credentials. +func (c *Client) SignIn(ctx context.Context) error { + return c.Session.SignIn(ctx, c.appleID, c.password) +} + +// IntoReader marshals the provided values into a JSON encoded reader +func IntoReader(values any) (*bytes.Reader, error) { + m, err := json.Marshal(values) + if err != nil { + return nil, err + } + return bytes.NewReader(m), nil +} + +// RequestError holds info on a result state, icloud can return a 200 but the result is unknown +type RequestError struct { + Status string + Text string +} + +// Error satisfy the error interface. +func (e *RequestError) Error() string { + return fmt.Sprintf("%s: %s", e.Text, e.Status) +} + +func newRequestError(Status string, Text string) *RequestError { + return &RequestError{ + Status: strings.ToLower(Status), + Text: Text, + } +} + +// newErr orf makes a new error from sprintf parameters. +func newRequestErrorf(Status string, Text string, Parameters ...interface{}) *RequestError { + return newRequestError(strings.ToLower(Status), fmt.Sprintf(Text, Parameters...)) +} diff --git a/backend/iclouddrive/api/drive.go b/backend/iclouddrive/api/drive.go new file mode 100644 index 000000000..5d735ebbd --- /dev/null +++ b/backend/iclouddrive/api/drive.go @@ -0,0 +1,913 @@ +package api + +import ( + "bytes" + "context" + "io" + "mime" + "net/http" + "net/url" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/lib/rest" +) + +const ( + defaultZone = "com.apple.CloudDocs" + statusOk = "OK" + statusEtagConflict = "ETAG_CONFLICT" +) + +// DriveService represents an iCloud Drive service. +type DriveService struct { + icloud *Client + RootID string + endpoint string + docsEndpoint string +} + +// NewDriveService creates a new DriveService instance. +func NewDriveService(icloud *Client) (*DriveService, error) { + return &DriveService{icloud: icloud, RootID: "FOLDER::com.apple.CloudDocs::root", endpoint: icloud.Session.AccountInfo.Webservices["drivews"].URL, docsEndpoint: icloud.Session.AccountInfo.Webservices["docws"].URL}, nil +} + +// GetItemByDriveID retrieves a DriveItem by its Drive ID. +func (d *DriveService) GetItemByDriveID(ctx context.Context, id string, includeChildren bool) (*DriveItem, *http.Response, error) { + items, resp, err := d.GetItemsByDriveID(ctx, []string{id}, includeChildren) + if err != nil { + return nil, resp, err + } + return items[0], resp, err +} + +// GetItemsByDriveID retrieves DriveItems by their Drive IDs. +func (d *DriveService) GetItemsByDriveID(ctx context.Context, ids []string, includeChildren bool) ([]*DriveItem, *http.Response, error) { + var err error + _items := []map[string]any{} + for _, id := range ids { + _items = append(_items, map[string]any{ + "drivewsid": id, + "partialData": false, + "includeHierarchy": false, + }) + } + + var body *bytes.Reader + var path string + if !includeChildren { + values := []map[string]any{{ + "items": _items, + }} + body, err = IntoReader(values) + if err != nil { + return nil, nil, err + } + path = "/retrieveItemDetails" + } else { + values := _items + body, err = IntoReader(values) + if err != nil { + return nil, nil, err + } + path = "/retrieveItemDetailsInFolders" + } + + opts := rest.Opts{ + Method: "POST", + Path: path, + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.endpoint, + Body: body, + } + var items []*DriveItem + resp, err := d.icloud.Request(ctx, opts, nil, &items) + if err != nil { + return nil, resp, err + } + + return items, resp, err +} + +// GetDocByPath retrieves a document by its path. +func (d *DriveService) GetDocByPath(ctx context.Context, path string) (*Document, *http.Response, error) { + values := url.Values{} + values.Set("unified_format", "false") + body, err := IntoReader(path) + if err != nil { + return nil, nil, err + } + opts := rest.Opts{ + Method: "POST", + Path: "/ws/" + defaultZone + "/list/lookup_by_path", + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.docsEndpoint, + Parameters: values, + Body: body, + } + var item []*Document + resp, err := d.icloud.Request(ctx, opts, nil, &item) + if err != nil { + return nil, resp, err + } + + return item[0], resp, err +} + +// GetItemByPath retrieves a DriveItem by its path. +func (d *DriveService) GetItemByPath(ctx context.Context, path string) (*DriveItem, *http.Response, error) { + values := url.Values{} + values.Set("unified_format", "true") + + body, err := IntoReader(path) + if err != nil { + return nil, nil, err + } + opts := rest.Opts{ + Method: "POST", + Path: "/ws/" + defaultZone + "/list/lookup_by_path", + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.docsEndpoint, + Parameters: values, + Body: body, + } + var item []*DriveItem + resp, err := d.icloud.Request(ctx, opts, nil, &item) + if err != nil { + return nil, resp, err + } + + return item[0], resp, err +} + +// GetDocByItemID retrieves a document by its item ID. +func (d *DriveService) GetDocByItemID(ctx context.Context, id string) (*Document, *http.Response, error) { + values := url.Values{} + values.Set("document_id", id) + values.Set("unified_format", "false") // important + opts := rest.Opts{ + Method: "GET", + Path: "/ws/" + defaultZone + "/list/lookup_by_id", + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.docsEndpoint, + Parameters: values, + } + var item *Document + resp, err := d.icloud.Request(ctx, opts, nil, &item) + if err != nil { + return nil, resp, err + } + + return item, resp, err +} + +// GetItemRawByItemID retrieves a DriveItemRaw by its item ID. +func (d *DriveService) GetItemRawByItemID(ctx context.Context, id string) (*DriveItemRaw, *http.Response, error) { + opts := rest.Opts{ + Method: "GET", + Path: "/v1/item/" + id, + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.docsEndpoint, + } + var item *DriveItemRaw + resp, err := d.icloud.Request(ctx, opts, nil, &item) + if err != nil { + return nil, resp, err + } + + return item, resp, err +} + +// GetItemsInFolder retrieves a list of DriveItemRaw objects in a folder with the given ID. +func (d *DriveService) GetItemsInFolder(ctx context.Context, id string, limit int64) ([]*DriveItemRaw, *http.Response, error) { + values := url.Values{} + values.Set("limit", strconv.FormatInt(limit, 10)) + + opts := rest.Opts{ + Method: "GET", + Path: "/v1/enumerate/" + id, + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.docsEndpoint, + Parameters: values, + } + + items := struct { + Items []*DriveItemRaw `json:"drive_item"` + }{} + + resp, err := d.icloud.Request(ctx, opts, nil, &items) + if err != nil { + return nil, resp, err + } + + return items.Items, resp, err +} + +// GetDownloadURLByDriveID retrieves the download URL for a file in the DriveService. +func (d *DriveService) GetDownloadURLByDriveID(ctx context.Context, id string) (string, *http.Response, error) { + _, zone, docid := DeconstructDriveID(id) + values := url.Values{} + values.Set("document_id", docid) + + if zone == "" { + zone = defaultZone + } + + opts := rest.Opts{ + Method: "GET", + Path: "/ws/" + zone + "/download/by_id", + Parameters: values, + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.docsEndpoint, + } + + var filer *FileRequest + resp, err := d.icloud.Request(ctx, opts, nil, &filer) + + if err != nil { + return "", resp, err + } + + var url string + if filer.DataToken != nil { + url = filer.DataToken.URL + } else { + url = filer.PackageToken.URL + } + + return url, resp, err +} + +// DownloadFile downloads a file from the given URL using the provided options. +func (d *DriveService) DownloadFile(ctx context.Context, url string, opt []fs.OpenOption) (*http.Response, error) { + opts := &rest.Opts{ + Method: "GET", + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: url, + Options: opt, + } + + resp, err := d.icloud.srv.Call(ctx, opts) + if err != nil { + // icloud has some weird http codes + if resp.StatusCode == 330 { + loc, err := resp.Location() + if err == nil { + return d.DownloadFile(ctx, loc.String(), opt) + } + } + + return resp, err + } + return d.icloud.srv.Call(ctx, opts) +} + +// MoveItemToTrashByItemID moves an item to the trash based on the item ID. +func (d *DriveService) MoveItemToTrashByItemID(ctx context.Context, id, etag string, force bool) (*DriveItem, *http.Response, error) { + doc, resp, err := d.GetDocByItemID(ctx, id) + if err != nil { + return nil, resp, err + } + return d.MoveItemToTrashByID(ctx, doc.DriveID(), etag, force) +} + +// MoveItemToTrashByID moves an item to the trash based on the item ID. +func (d *DriveService) MoveItemToTrashByID(ctx context.Context, drivewsid, etag string, force bool) (*DriveItem, *http.Response, error) { + values := map[string]any{ + "items": []map[string]any{{ + "drivewsid": drivewsid, + "etag": etag, + "clientId": drivewsid, + }}} + + body, err := IntoReader(values) + if err != nil { + return nil, nil, err + } + + opts := rest.Opts{ + Method: "POST", + Path: "/moveItemsToTrash", + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.endpoint, + Body: body, + } + + item := struct { + Items []*DriveItem `json:"items"` + }{} + resp, err := d.icloud.Request(ctx, opts, nil, &item) + + if err != nil { + return nil, resp, err + } + + if item.Items[0].Status != statusOk { + // rerun with latest etag + if force && item.Items[0].Status == "ETAG_CONFLICT" { + return d.MoveItemToTrashByID(ctx, drivewsid, item.Items[0].Etag, false) + } + + err = newRequestError(item.Items[0].Status, "unknown request status") + } + + return item.Items[0], resp, err +} + +// CreateNewFolderByItemID creates a new folder by item ID. +func (d *DriveService) CreateNewFolderByItemID(ctx context.Context, id, name string) (*DriveItem, *http.Response, error) { + doc, resp, err := d.GetDocByItemID(ctx, id) + if err != nil { + return nil, resp, err + } + return d.CreateNewFolderByDriveID(ctx, doc.DriveID(), name) +} + +// CreateNewFolderByDriveID creates a new folder by its Drive ID. +func (d *DriveService) CreateNewFolderByDriveID(ctx context.Context, drivewsid, name string) (*DriveItem, *http.Response, error) { + values := map[string]any{ + "destinationDrivewsId": drivewsid, + "folders": []map[string]any{{ + "clientId": "FOLDER::UNKNOWN_ZONE::TempId-" + uuid.New().String(), + "name": name, + }}, + } + + body, err := IntoReader(values) + if err != nil { + return nil, nil, err + } + + opts := rest.Opts{ + Method: "POST", + Path: "/createFolders", + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.endpoint, + Body: body, + } + var fResp *CreateFoldersResponse + resp, err := d.icloud.Request(ctx, opts, nil, &fResp) + if err != nil { + return nil, resp, err + } + status := fResp.Folders[0].Status + if status != statusOk { + err = newRequestError(status, "unknown request status") + } + + return fResp.Folders[0], resp, err +} + +// RenameItemByItemID renames a DriveItem by its item ID. +func (d *DriveService) RenameItemByItemID(ctx context.Context, id, etag, name string, force bool) (*DriveItem, *http.Response, error) { + doc, resp, err := d.GetDocByItemID(ctx, id) + if err != nil { + return nil, resp, err + } + return d.RenameItemByDriveID(ctx, doc.DriveID(), doc.Etag, name, force) +} + +// RenameItemByDriveID renames a DriveItem by its drive ID. +func (d *DriveService) RenameItemByDriveID(ctx context.Context, id, etag, name string, force bool) (*DriveItem, *http.Response, error) { + values := map[string]any{ + "items": []map[string]any{{ + "drivewsid": id, + "name": name, + "etag": etag, + // "extension": split[1], + }}, + } + + body, err := IntoReader(values) + if err != nil { + return nil, nil, err + } + + opts := rest.Opts{ + Method: "POST", + Path: "/renameItems", + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.endpoint, + Body: body, + } + var items *DriveItem + resp, err := d.icloud.Request(ctx, opts, nil, &items) + + if err != nil { + return nil, resp, err + } + + status := items.Items[0].Status + if status != statusOk { + // rerun with latest etag + if force && status == "ETAG_CONFLICT" { + return d.RenameItemByDriveID(ctx, id, items.Items[0].Etag, name, false) + } + err = newRequestErrorf(status, "unknown inner status for: %s %s", opts.Method, resp.Request.URL) + } + + return items.Items[0], resp, err +} + +// MoveItemByItemID moves an item by its item ID to a destination item ID. +func (d *DriveService) MoveItemByItemID(ctx context.Context, id, etag, dstID string, force bool) (*DriveItem, *http.Response, error) { + docSrc, resp, err := d.GetDocByItemID(ctx, id) + if err != nil { + return nil, resp, err + } + docDst, resp, err := d.GetDocByItemID(ctx, dstID) + if err != nil { + return nil, resp, err + } + return d.MoveItemByDriveID(ctx, docSrc.DriveID(), docSrc.Etag, docDst.DriveID(), force) +} + +// MoveItemByDocID moves an item by its doc ID. +// func (d *DriveService) MoveItemByDocID(ctx context.Context, srcDocID, srcEtag, dstDocID string, force bool) (*DriveItem, *http.Response, error) { +// return d.MoveItemByDriveID(ctx, srcDocID, srcEtag, docDst.DriveID(), force) +// } + +// MoveItemByDriveID moves an item by its drive ID. +func (d *DriveService) MoveItemByDriveID(ctx context.Context, id, etag, dstID string, force bool) (*DriveItem, *http.Response, error) { + values := map[string]any{ + "destinationDrivewsId": dstID, + "items": []map[string]any{{ + "drivewsid": id, + "etag": etag, + "clientId": id, + }}, + } + + body, err := IntoReader(values) + if err != nil { + return nil, nil, err + } + + opts := rest.Opts{ + Method: "POST", + Path: "/moveItems", + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.endpoint, + Body: body, + } + + var items *DriveItem + resp, err := d.icloud.Request(ctx, opts, nil, &items) + + if err != nil { + return nil, resp, err + } + + status := items.Items[0].Status + if status != statusOk { + // rerun with latest etag + if force && status == "ETAG_CONFLICT" { + return d.MoveItemByDriveID(ctx, id, items.Items[0].Etag, dstID, false) + } + err = newRequestErrorf(status, "unknown inner status for: %s %s", opts.Method, resp.Request.URL) + } + + return items.Items[0], resp, err +} + +// CopyDocByItemID copies a document by its item ID. +func (d *DriveService) CopyDocByItemID(ctx context.Context, itemID string) (*DriveItemRaw, *http.Response, error) { + // putting name in info doesnt work. extension does work so assume this is a bug in the endpoint + values := map[string]any{ + "info_to_update": map[string]any{}, + } + + body, err := IntoReader(values) + if err != nil { + return nil, nil, err + } + opts := rest.Opts{ + Method: "POST", + Path: "/v1/item/copy/" + itemID, + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.docsEndpoint, + Body: body, + } + + var info *DriveItemRaw + resp, err := d.icloud.Request(ctx, opts, nil, &info) + if err != nil { + return nil, resp, err + } + return info, resp, err +} + +// CreateUpload creates an url for an upload. +func (d *DriveService) CreateUpload(ctx context.Context, size int64, name string) (*UploadResponse, *http.Response, error) { + // first we need to request an upload url + values := map[string]any{ + "filename": name, + "type": "FILE", + "size": strconv.FormatInt(size, 10), + "content_type": GetContentTypeForFile(name), + } + body, err := IntoReader(values) + if err != nil { + return nil, nil, err + } + + opts := rest.Opts{ + Method: "POST", + Path: "/ws/" + defaultZone + "/upload/web", + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.docsEndpoint, + Body: body, + } + var responseInfo []*UploadResponse + resp, err := d.icloud.Request(ctx, opts, nil, &responseInfo) + if err != nil { + return nil, resp, err + } + return responseInfo[0], resp, err +} + +// Upload uploads a file to the given url +func (d *DriveService) Upload(ctx context.Context, in io.Reader, size int64, name, uploadURL string) (*SingleFileResponse, *http.Response, error) { + // TODO: implement multipart upload + opts := rest.Opts{ + Method: "POST", + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: uploadURL, + Body: in, + ContentLength: &size, + ContentType: GetContentTypeForFile(name), + // MultipartContentName: "files", + MultipartFileName: name, + } + var singleFileResponse *SingleFileResponse + resp, err := d.icloud.Request(ctx, opts, nil, &singleFileResponse) + if err != nil { + return nil, resp, err + } + return singleFileResponse, resp, err +} + +// UpdateFile updates a file in the DriveService. +// +// ctx: the context.Context object for the request. +// r: a pointer to the UpdateFileInfo struct containing the information for the file update. +// Returns a pointer to the DriveItem struct representing the updated file, the http.Response object, and an error if any. +func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo) (*DriveItem, *http.Response, error) { + body, err := IntoReader(r) + if err != nil { + return nil, nil, err + } + opts := rest.Opts{ + Method: "POST", + Path: "/ws/" + defaultZone + "/update/documents", + ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}), + RootURL: d.docsEndpoint, + Body: body, + } + var responseInfo *DocumentUpdateResponse + resp, err := d.icloud.Request(ctx, opts, nil, &responseInfo) + if err != nil { + return nil, resp, err + } + + doc := responseInfo.Results[0].Document + item := DriveItem{ + Drivewsid: "FILE::com.apple.CloudDocs::" + doc.DocumentID, + Docwsid: doc.DocumentID, + Itemid: doc.ItemID, + Etag: doc.Etag, + ParentID: doc.ParentID, + DateModified: time.Unix(r.Mtime, 0), + DateCreated: time.Unix(r.Mtime, 0), + Type: doc.Type, + Name: doc.Name, + Size: doc.Size, + } + + return &item, resp, err +} + +// UpdateFileInfo represents the information for an update to a file in the DriveService. +type UpdateFileInfo struct { + AllowConflict bool `json:"allow_conflict"` + Btime int64 `json:"btime"` + Command string `json:"command"` + CreateShortGUID bool `json:"create_short_guid"` + Data struct { + Receipt string `json:"receipt,omitempty"` + ReferenceSignature string `json:"reference_signature,omitempty"` + Signature string `json:"signature,omitempty"` + Size int64 `json:"size,omitempty"` + WrappingKey string `json:"wrapping_key,omitempty"` + } `json:"data,omitempty"` + DocumentID string `json:"document_id"` + FileFlags FileFlags `json:"file_flags"` + Mtime int64 `json:"mtime"` + Path struct { + Path string `json:"path"` + StartingDocumentID string `json:"starting_document_id"` + } `json:"path"` +} + +// FileFlags defines the file flags for a document. +type FileFlags struct { + IsExecutable bool `json:"is_executable"` + IsHidden bool `json:"is_hidden"` + IsWritable bool `json:"is_writable"` +} + +// NewUpdateFileInfo creates a new UpdateFileInfo object with default values. +// +// Returns an UpdateFileInfo object. +func NewUpdateFileInfo() UpdateFileInfo { + return UpdateFileInfo{ + Command: "add_file", + CreateShortGUID: true, + AllowConflict: true, + FileFlags: FileFlags{ + IsExecutable: true, + IsHidden: false, + IsWritable: false, + }, + } +} + +// DriveItemRaw is a raw drive item. +// not suure what to call this but there seems to be a "unified" and non "unified" drive item response. This is the non unified. +type DriveItemRaw struct { + ItemID string `json:"item_id"` + ItemInfo *DriveItemRawInfo `json:"item_info"` +} + +// SplitName splits the name of a DriveItemRaw into its name and extension. +// +// It returns the name and extension as separate strings. If the name ends with a dot, +// it means there is no extension, so an empty string is returned for the extension. +// If the name does not contain a dot, it means +func (d *DriveItemRaw) SplitName() (string, string) { + name := d.ItemInfo.Name + // ends with a dot, no extension + if strings.HasSuffix(name, ".") { + return name, "" + } + lastInd := strings.LastIndex(name, ".") + + if lastInd == -1 { + return name, "" + } + return name[:lastInd], name[lastInd+1:] +} + +// ModTime returns the modification time of the DriveItemRaw. +// +// It parses the ModifiedAt field of the ItemInfo struct and converts it to a time.Time value. +// If the parsing fails, it returns the zero value of time.Time. +// The returned time.Time value represents the modification time of the DriveItemRaw. +func (d *DriveItemRaw) ModTime() time.Time { + i, err := strconv.ParseInt(d.ItemInfo.ModifiedAt, 10, 64) + if err != nil { + return time.Time{} + } + return time.UnixMilli(i) +} + +// CreatedTime returns the creation time of the DriveItemRaw. +// +// It parses the CreatedAt field of the ItemInfo struct and converts it to a time.Time value. +// If the parsing fails, it returns the zero value of time.Time. +// The returned time.Time +func (d *DriveItemRaw) CreatedTime() time.Time { + i, err := strconv.ParseInt(d.ItemInfo.CreatedAt, 10, 64) + if err != nil { + return time.Time{} + } + return time.UnixMilli(i) +} + +// DriveItemRawInfo is the raw information about a drive item. +type DriveItemRawInfo struct { + Name string `json:"name"` + // Extension is absolutely borked on endpoints so dont use it. + Extension string `json:"extension"` + Size int64 `json:"size,string"` + Type string `json:"type"` + Version string `json:"version"` + ModifiedAt string `json:"modified_at"` + CreatedAt string `json:"created_at"` + Urls struct { + URLDownload string `json:"url_download"` + } `json:"urls"` +} + +// IntoDriveItem converts a DriveItemRaw into a DriveItem. +// +// It takes no parameters. +// It returns a pointer to a DriveItem. +func (d *DriveItemRaw) IntoDriveItem() *DriveItem { + name, extension := d.SplitName() + return &DriveItem{ + Itemid: d.ItemID, + Name: name, + Extension: extension, + Type: d.ItemInfo.Type, + Etag: d.ItemInfo.Version, + DateModified: d.ModTime(), + DateCreated: d.CreatedTime(), + Size: d.ItemInfo.Size, + Urls: d.ItemInfo.Urls, + } +} + +// DocumentUpdateResponse is the response of a document update request. +type DocumentUpdateResponse struct { + Status struct { + StatusCode int `json:"status_code"` + ErrorMessage string `json:"error_message"` + } `json:"status"` + Results []struct { + Status struct { + StatusCode int `json:"status_code"` + ErrorMessage string `json:"error_message"` + } `json:"status"` + OperationID interface{} `json:"operation_id"` + Document *Document `json:"document"` + } `json:"results"` +} + +// Document represents a document on iCloud. +type Document struct { + Status struct { + StatusCode int `json:"status_code"` + ErrorMessage string `json:"error_message"` + } `json:"status"` + DocumentID string `json:"document_id"` + ItemID string `json:"item_id"` + Urls struct { + URLDownload string `json:"url_download"` + } `json:"urls"` + Etag string `json:"etag"` + ParentID string `json:"parent_id"` + Name string `json:"name"` + Type string `json:"type"` + Deleted bool `json:"deleted"` + Mtime int64 `json:"mtime"` + LastEditorName string `json:"last_editor_name"` + Data DocumentData `json:"data"` + Size int64 `json:"size"` + Btime int64 `json:"btime"` + Zone string `json:"zone"` + FileFlags struct { + IsExecutable bool `json:"is_executable"` + IsWritable bool `json:"is_writable"` + IsHidden bool `json:"is_hidden"` + } `json:"file_flags"` + LastOpenedTime int64 `json:"lastOpenedTime"` + RestorePath interface{} `json:"restorePath"` + HasChainedParent bool `json:"hasChainedParent"` +} + +// DriveID returns the drive ID of the Document. +func (d *Document) DriveID() string { + if d.Zone == "" { + d.Zone = defaultZone + } + return d.Type + "::" + d.Zone + "::" + d.DocumentID +} + +// DocumentData represents the data of a document. +type DocumentData struct { + Signature string `json:"signature"` + Owner string `json:"owner"` + Size int64 `json:"size"` + ReferenceSignature string `json:"reference_signature"` + WrappingKey string `json:"wrapping_key"` + PcsInfo string `json:"pcsInfo"` +} + +// SingleFileResponse is the response of a single file request. +type SingleFileResponse struct { + SingleFile *SingleFileInfo `json:"singleFile"` +} + +// SingleFileInfo represents the information of a single file. +type SingleFileInfo struct { + ReferenceSignature string `json:"referenceChecksum"` + Size int64 `json:"size"` + Signature string `json:"fileChecksum"` + WrappingKey string `json:"wrappingKey"` + Receipt string `json:"receipt"` +} + +// UploadResponse is the response of an upload request. +type UploadResponse struct { + URL string `json:"url"` + DocumentID string `json:"document_id"` +} + +// FileRequestToken represents the token of a file request. +type FileRequestToken struct { + URL string `json:"url"` + Token string `json:"token"` + Signature string `json:"signature"` + WrappingKey string `json:"wrapping_key"` + ReferenceSignature string `json:"reference_signature"` +} + +// FileRequest represents the request of a file. +type FileRequest struct { + DocumentID string `json:"document_id"` + ItemID string `json:"item_id"` + OwnerDsid int64 `json:"owner_dsid"` + DataToken *FileRequestToken `json:"data_token,omitempty"` + PackageToken *FileRequestToken `json:"package_token,omitempty"` + DoubleEtag string `json:"double_etag"` +} + +// CreateFoldersResponse is the response of a create folders request. +type CreateFoldersResponse struct { + Folders []*DriveItem `json:"folders"` +} + +// DriveItem represents an item on iCloud. +type DriveItem struct { + DateCreated time.Time `json:"dateCreated"` + Drivewsid string `json:"drivewsid"` + Docwsid string `json:"docwsid"` + Itemid string `json:"item_id"` + Zone string `json:"zone"` + Name string `json:"name"` + ParentID string `json:"parentId"` + Hierarchy []DriveItem `json:"hierarchy"` + Etag string `json:"etag"` + Type string `json:"type"` + AssetQuota int64 `json:"assetQuota"` + FileCount int64 `json:"fileCount"` + ShareCount int64 `json:"shareCount"` + ShareAliasCount int64 `json:"shareAliasCount"` + DirectChildrenCount int64 `json:"directChildrenCount"` + Items []*DriveItem `json:"items"` + NumberOfItems int64 `json:"numberOfItems"` + Status string `json:"status"` + Extension string `json:"extension,omitempty"` + DateModified time.Time `json:"dateModified,omitempty"` + DateChanged time.Time `json:"dateChanged,omitempty"` + Size int64 `json:"size,omitempty"` + LastOpenTime time.Time `json:"lastOpenTime,omitempty"` + Urls struct { + URLDownload string `json:"url_download"` + } `json:"urls"` +} + +// IsFolder returns true if the item is a folder. +func (d *DriveItem) IsFolder() bool { + return d.Type == "FOLDER" || d.Type == "APP_CONTAINER" || d.Type == "APP_LIBRARY" +} + +// DownloadURL returns the download URL of the item. +func (d *DriveItem) DownloadURL() string { + return d.Urls.URLDownload +} + +// FullName returns the full name of the item. +// name + extension +func (d *DriveItem) FullName() string { + if d.Extension != "" { + return d.Name + "." + d.Extension + } + return d.Name +} + +// GetDocIDFromDriveID returns the DocumentID from the drive ID. +func GetDocIDFromDriveID(id string) string { + split := strings.Split(id, "::") + return split[len(split)-1] +} + +// DeconstructDriveID returns the document type, zone, and document ID from the drive ID. +func DeconstructDriveID(id string) (docType, zone, docid string) { + split := strings.Split(id, "::") + if len(split) < 3 { + return "", "", id + } + return split[0], split[1], split[2] +} + +// ConstructDriveID constructs a drive ID from the given components. +func ConstructDriveID(id string, zone string, t string) string { + return strings.Join([]string{t, zone, id}, "::") +} + +// GetContentTypeForFile detects content type for given file name. +func GetContentTypeForFile(name string) string { + // detect MIME type by looking at the filename only + mimeType := mime.TypeByExtension(filepath.Ext(name)) + if mimeType == "" { + // api requires a mime type passed in + mimeType = "text/plain" + } + return strings.Split(mimeType, ";")[0] +} diff --git a/backend/iclouddrive/api/session.go b/backend/iclouddrive/api/session.go new file mode 100644 index 000000000..7ee350675 --- /dev/null +++ b/backend/iclouddrive/api/session.go @@ -0,0 +1,412 @@ +package api + +import ( + "context" + "fmt" + "net/http" + "net/url" + "slices" + "strings" + + "github.com/oracle/oci-go-sdk/v65/common" + + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/lib/rest" +) + +// Session represents an iCloud session +type Session struct { + SessionToken string `json:"session_token"` + Scnt string `json:"scnt"` + SessionID string `json:"session_id"` + AccountCountry string `json:"account_country"` + TrustToken string `json:"trust_token"` + ClientID string `json:"client_id"` + Cookies []*http.Cookie `json:"cookies"` + AccountInfo AccountInfo `json:"account_info"` + + srv *rest.Client `json:"-"` +} + +// String returns the session as a string +// func (s *Session) String() string { +// jsession, _ := json.Marshal(s) +// return string(jsession) +// } + +// Request makes a request +func (s *Session) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (*http.Response, error) { + resp, err := s.srv.CallJSON(ctx, &opts, &request, &response) + + if err != nil { + return resp, err + } + + if val := resp.Header.Get("X-Apple-ID-Account-Country"); val != "" { + s.AccountCountry = val + } + if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" { + s.SessionID = val + } + if val := resp.Header.Get("X-Apple-Session-Token"); val != "" { + s.SessionToken = val + } + if val := resp.Header.Get("X-Apple-TwoSV-Trust-Token"); val != "" { + s.TrustToken = val + } + if val := resp.Header.Get("scnt"); val != "" { + s.Scnt = val + } + + return resp, nil +} + +// Requires2FA returns true if the session requires 2FA +func (s *Session) Requires2FA() bool { + return s.AccountInfo.DsInfo.HsaVersion == 2 && s.AccountInfo.HsaChallengeRequired +} + +// SignIn signs in the session +func (s *Session) SignIn(ctx context.Context, appleID, password string) error { + trustTokens := []string{} + if s.TrustToken != "" { + trustTokens = []string{s.TrustToken} + } + values := map[string]any{ + "accountName": appleID, + "password": password, + "rememberMe": true, + "trustTokens": trustTokens, + } + body, err := IntoReader(values) + if err != nil { + return err + } + opts := rest.Opts{ + Method: "POST", + Path: "/signin", + Parameters: url.Values{}, + ExtraHeaders: s.GetAuthHeaders(map[string]string{}), + RootURL: authEndpoint, + IgnoreStatus: true, // need to handle 409 for hsa2 + NoResponse: true, + Body: body, + } + opts.Parameters.Set("isRememberMeEnabled", "true") + _, err = s.Request(ctx, opts, nil, nil) + + return err + +} + +// AuthWithToken authenticates the session +func (s *Session) AuthWithToken(ctx context.Context) error { + values := map[string]any{ + "accountCountryCode": s.AccountCountry, + "dsWebAuthToken": s.SessionToken, + "extended_login": true, + "trustToken": s.TrustToken, + } + body, err := IntoReader(values) + if err != nil { + return err + } + opts := rest.Opts{ + Method: "POST", + Path: "/accountLogin", + ExtraHeaders: GetCommonHeaders(map[string]string{}), + RootURL: setupEndpoint, + Body: body, + } + + resp, err := s.Request(ctx, opts, nil, &s.AccountInfo) + if err == nil { + s.Cookies = resp.Cookies() + } + + return err +} + +// Validate2FACode validates the 2FA code +func (s *Session) Validate2FACode(ctx context.Context, code string) error { + values := map[string]interface{}{"securityCode": map[string]string{"code": code}} + body, err := IntoReader(values) + if err != nil { + return err + } + + headers := s.GetAuthHeaders(map[string]string{}) + headers["scnt"] = s.Scnt + headers["X-Apple-ID-Session-Id"] = s.SessionID + + opts := rest.Opts{ + Method: "POST", + Path: "/verify/trusteddevice/securitycode", + ExtraHeaders: headers, + RootURL: authEndpoint, + Body: body, + NoResponse: true, + } + + _, err = s.Request(ctx, opts, nil, nil) + if err == nil { + if err := s.TrustSession(ctx); err != nil { + return err + } + + return nil + } + + return fmt.Errorf("validate2FACode failed: %w", err) +} + +// TrustSession trusts the session +func (s *Session) TrustSession(ctx context.Context) error { + headers := s.GetAuthHeaders(map[string]string{}) + headers["scnt"] = s.Scnt + headers["X-Apple-ID-Session-Id"] = s.SessionID + + opts := rest.Opts{ + Method: "GET", + Path: "/2sv/trust", + ExtraHeaders: headers, + RootURL: authEndpoint, + NoResponse: true, + ContentLength: common.Int64(0), + } + + _, err := s.Request(ctx, opts, nil, nil) + if err != nil { + return fmt.Errorf("trustSession failed: %w", err) + } + + return s.AuthWithToken(ctx) +} + +// ValidateSession validates the session +func (s *Session) ValidateSession(ctx context.Context) error { + opts := rest.Opts{ + Method: "POST", + Path: "/validate", + ExtraHeaders: s.GetHeaders(map[string]string{}), + RootURL: setupEndpoint, + ContentLength: common.Int64(0), + } + _, err := s.Request(ctx, opts, nil, &s.AccountInfo) + if err != nil { + return fmt.Errorf("validateSession failed: %w", err) + } + + return nil +} + +// GetAuthHeaders returns the authentication headers for the session. +// +// It takes an `overwrite` map[string]string parameter which allows +// overwriting the default headers. It returns a map[string]string. +func (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string { + headers := map[string]string{ + "Accept": "application/json", + "Content-Type": "application/json", + "X-Apple-OAuth-Client-Id": s.ClientID, + "X-Apple-OAuth-Client-Type": "firstPartyAuth", + "X-Apple-OAuth-Redirect-URI": "https://www.icloud.com", + "X-Apple-OAuth-Require-Grant-Code": "true", + "X-Apple-OAuth-Response-Mode": "web_message", + "X-Apple-OAuth-Response-Type": "code", + "X-Apple-OAuth-State": s.ClientID, + "X-Apple-Widget-Key": s.ClientID, + "Origin": homeEndpoint, + "Referer": fmt.Sprintf("%s/", homeEndpoint), + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0", + } + for k, v := range overwrite { + headers[k] = v + } + return headers +} + +// GetHeaders Gets the authentication headers required for a request +func (s *Session) GetHeaders(overwrite map[string]string) map[string]string { + headers := GetCommonHeaders(map[string]string{}) + headers["Cookie"] = s.GetCookieString() + for k, v := range overwrite { + headers[k] = v + } + return headers +} + +// GetCookieString returns the cookie header string for the session. +func (s *Session) GetCookieString() string { + cookieHeader := "" + // we only care about name and value. + for _, cookie := range s.Cookies { + cookieHeader = cookieHeader + cookie.Name + "=" + cookie.Value + ";" + } + return cookieHeader +} + +// GetCommonHeaders generates common HTTP headers with optional overwrite. +func GetCommonHeaders(overwrite map[string]string) map[string]string { + headers := map[string]string{ + "Content-Type": "application/json", + "Origin": baseEndpoint, + "Referer": fmt.Sprintf("%s/", baseEndpoint), + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0", + } + for k, v := range overwrite { + headers[k] = v + } + return headers +} + +// MergeCookies merges two slices of http.Cookies, ensuring no duplicates are added. +func MergeCookies(left []*http.Cookie, right []*http.Cookie) ([]*http.Cookie, error) { + var hashes []string + for _, cookie := range right { + hashes = append(hashes, cookie.Raw) + } + for _, cookie := range left { + if !slices.Contains(hashes, cookie.Raw) { + right = append(right, cookie) + } + } + return right, nil +} + +// GetCookiesForDomain filters the provided cookies based on the domain of the given URL. +func GetCookiesForDomain(url *url.URL, cookies []*http.Cookie) ([]*http.Cookie, error) { + var domainCookies []*http.Cookie + for _, cookie := range cookies { + if strings.HasSuffix(url.Host, cookie.Domain) { + domainCookies = append(domainCookies, cookie) + } + } + return domainCookies, nil +} + +// NewSession creates a new Session instance with default values. +func NewSession() *Session { + session := &Session{} + session.srv = rest.NewClient(fshttp.NewClient(context.Background())).SetRoot(baseEndpoint) + //session.ClientID = "auth-" + uuid.New().String() + return session +} + +// AccountInfo represents an account info +type AccountInfo struct { + DsInfo *ValidateDataDsInfo `json:"dsInfo"` + HasMinimumDeviceForPhotosWeb bool `json:"hasMinimumDeviceForPhotosWeb"` + ICDPEnabled bool `json:"iCDPEnabled"` + Webservices map[string]*webService `json:"webservices"` + PcsEnabled bool `json:"pcsEnabled"` + TermsUpdateNeeded bool `json:"termsUpdateNeeded"` + ConfigBag struct { + Urls struct { + AccountCreateUI string `json:"accountCreateUI"` + AccountLoginUI string `json:"accountLoginUI"` + AccountLogin string `json:"accountLogin"` + AccountRepairUI string `json:"accountRepairUI"` + DownloadICloudTerms string `json:"downloadICloudTerms"` + RepairDone string `json:"repairDone"` + AccountAuthorizeUI string `json:"accountAuthorizeUI"` + VettingURLForEmail string `json:"vettingUrlForEmail"` + AccountCreate string `json:"accountCreate"` + GetICloudTerms string `json:"getICloudTerms"` + VettingURLForPhone string `json:"vettingUrlForPhone"` + } `json:"urls"` + AccountCreateEnabled bool `json:"accountCreateEnabled"` + } `json:"configBag"` + HsaTrustedBrowser bool `json:"hsaTrustedBrowser"` + AppsOrder []string `json:"appsOrder"` + Version int `json:"version"` + IsExtendedLogin bool `json:"isExtendedLogin"` + PcsServiceIdentitiesIncluded bool `json:"pcsServiceIdentitiesIncluded"` + IsRepairNeeded bool `json:"isRepairNeeded"` + HsaChallengeRequired bool `json:"hsaChallengeRequired"` + RequestInfo struct { + Country string `json:"country"` + TimeZone string `json:"timeZone"` + Region string `json:"region"` + } `json:"requestInfo"` + PcsDeleted bool `json:"pcsDeleted"` + ICloudInfo struct { + SafariBookmarksHasMigratedToCloudKit bool `json:"SafariBookmarksHasMigratedToCloudKit"` + } `json:"iCloudInfo"` + Apps map[string]*ValidateDataApp `json:"apps"` +} + +// ValidateDataDsInfo represents an validation info +type ValidateDataDsInfo struct { + HsaVersion int `json:"hsaVersion"` + LastName string `json:"lastName"` + ICDPEnabled bool `json:"iCDPEnabled"` + TantorMigrated bool `json:"tantorMigrated"` + Dsid string `json:"dsid"` + HsaEnabled bool `json:"hsaEnabled"` + IsHideMyEmailSubscriptionActive bool `json:"isHideMyEmailSubscriptionActive"` + IroncadeMigrated bool `json:"ironcadeMigrated"` + Locale string `json:"locale"` + BrZoneConsolidated bool `json:"brZoneConsolidated"` + ICDRSCapableDeviceList string `json:"ICDRSCapableDeviceList"` + IsManagedAppleID bool `json:"isManagedAppleID"` + IsCustomDomainsFeatureAvailable bool `json:"isCustomDomainsFeatureAvailable"` + IsHideMyEmailFeatureAvailable bool `json:"isHideMyEmailFeatureAvailable"` + ContinueOnDeviceEligibleDeviceInfo []string `json:"ContinueOnDeviceEligibleDeviceInfo"` + Gilligvited bool `json:"gilligvited"` + AppleIDAliases []interface{} `json:"appleIdAliases"` + UbiquityEOLEnabled bool `json:"ubiquityEOLEnabled"` + IsPaidDeveloper bool `json:"isPaidDeveloper"` + CountryCode string `json:"countryCode"` + NotificationID string `json:"notificationId"` + PrimaryEmailVerified bool `json:"primaryEmailVerified"` + ADsID string `json:"aDsID"` + Locked bool `json:"locked"` + ICDRSCapableDeviceCount int `json:"ICDRSCapableDeviceCount"` + HasICloudQualifyingDevice bool `json:"hasICloudQualifyingDevice"` + PrimaryEmail string `json:"primaryEmail"` + AppleIDEntries []struct { + IsPrimary bool `json:"isPrimary"` + Type string `json:"type"` + Value string `json:"value"` + } `json:"appleIdEntries"` + GilliganEnabled bool `json:"gilligan-enabled"` + IsWebAccessAllowed bool `json:"isWebAccessAllowed"` + FullName string `json:"fullName"` + MailFlags struct { + IsThreadingAvailable bool `json:"isThreadingAvailable"` + IsSearchV2Provisioned bool `json:"isSearchV2Provisioned"` + SCKMail bool `json:"sCKMail"` + IsMppSupportedInCurrentCountry bool `json:"isMppSupportedInCurrentCountry"` + } `json:"mailFlags"` + LanguageCode string `json:"languageCode"` + AppleID string `json:"appleId"` + HasUnreleasedOS bool `json:"hasUnreleasedOS"` + AnalyticsOptInStatus bool `json:"analyticsOptInStatus"` + FirstName string `json:"firstName"` + ICloudAppleIDAlias string `json:"iCloudAppleIdAlias"` + NotesMigrated bool `json:"notesMigrated"` + BeneficiaryInfo struct { + IsBeneficiary bool `json:"isBeneficiary"` + } `json:"beneficiaryInfo"` + HasPaymentInfo bool `json:"hasPaymentInfo"` + PcsDelet bool `json:"pcsDelet"` + AppleIDAlias string `json:"appleIdAlias"` + BrMigrated bool `json:"brMigrated"` + StatusCode int `json:"statusCode"` + FamilyEligible bool `json:"familyEligible"` +} + +// ValidateDataApp represents an app +type ValidateDataApp struct { + CanLaunchWithOneFactor bool `json:"canLaunchWithOneFactor"` + IsQualifiedForBeta bool `json:"isQualifiedForBeta"` +} + +// WebService represents a web service +type webService struct { + PcsRequired bool `json:"pcsRequired"` + URL string `json:"url"` + UploadURL string `json:"uploadUrl"` + Status string `json:"status"` +} diff --git a/backend/iclouddrive/iclouddrive.go b/backend/iclouddrive/iclouddrive.go new file mode 100644 index 000000000..591d81718 --- /dev/null +++ b/backend/iclouddrive/iclouddrive.go @@ -0,0 +1,1174 @@ +//go:build !plan9 && !solaris + +// Package iclouddrive implements the iCloud Drive backend +package iclouddrive + +import ( + "bytes" + "context" + "path" + + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/fserrors" + + "github.com/rclone/rclone/backend/iclouddrive/api" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/dircache" + "github.com/rclone/rclone/lib/encoder" + "github.com/rclone/rclone/lib/pacer" +) + +/* +- dirCache operates on relative path to root +- path sanitization + - rule of thumb: sanitize before use, but store things as-is + - the paths cached in dirCache are after sanitizing + - the remote/dir passed in aren't, and are stored as-is +*/ + +const ( + configAppleID = "apple_id" + configPassword = "password" + configClientID = "client_id" + configCookies = "cookies" + configTrustToken = "trust_token" + + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "iclouddrive", + Description: "iCloud Drive", + Config: Config, + NewFs: NewFs, + Options: []fs.Option{{ + Name: configAppleID, + Help: "Apple ID.", + Required: true, + Sensitive: true, + }, { + Name: configPassword, + Help: "Password.", + Required: true, + IsPassword: true, + Sensitive: true, + }, { + Name: configTrustToken, + Help: "Trust token (internal use)", + IsPassword: false, + Required: false, + Sensitive: true, + Hide: fs.OptionHideBoth, + }, { + Name: configCookies, + Help: "cookies (internal use only)", + Required: false, + Advanced: false, + Sensitive: true, + Hide: fs.OptionHideBoth, + }, { + Name: configClientID, + Help: "Client id", + Required: false, + Advanced: true, + Default: "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d", + }, { + Name: config.ConfigEncoding, + Help: config.ConfigEncodingHelp, + Advanced: true, + Default: (encoder.Display | + //encoder.EncodeDot | + encoder.EncodeBackSlash | + encoder.EncodeInvalidUtf8), + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + AppleID string `config:"apple_id"` + Password string `config:"password"` + Photos bool `config:"photos"` + TrustToken string `config:"trust_token"` + Cookies string `config:"cookies"` + ClientID string `config:"client_id"` + Enc encoder.MultiEncoder `config:"encoding"` +} + +// Fs represents a remote icloud drive +type Fs struct { + name string // name of this remote + root string // the path we are working on. + rootID string + opt Options // parsed config options + features *fs.Features // optional features + dirCache *dircache.DirCache // Map of directory path to directory id + icloud *api.Client + service *api.DriveService + pacer *fs.Pacer // pacer for API calls +} + +// Object describes an icloud drive object +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path (relative to the fs.root) + size int64 // size of the object (on server, after encryption) + modTime time.Time // modification time of the object + createdTime time.Time // creation time of the object + driveID string // item ID of the object + docID string // document ID of the object + itemID string // item ID of the object + etag string + downloadURL string +} + +// Config configures the iCloud remote. +func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { + var err error + appleid, _ := m.Get(configAppleID) + if appleid == "" { + return nil, errors.New("a apple ID is required") + } + + password, _ := m.Get(configPassword) + if password != "" { + password, err = obscure.Reveal(password) + if err != nil { + return nil, err + } + } + + trustToken, _ := m.Get(configTrustToken) + cookieRaw, _ := m.Get(configCookies) + clientID, _ := m.Get(configClientID) + cookies := ReadCookies(cookieRaw) + + switch config.State { + case "": + icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil) + if err != nil { + return nil, err + } + if err := icloud.Authenticate(ctx); err != nil { + return nil, err + } + m.Set(configCookies, icloud.Session.GetCookieString()) + if icloud.Session.Requires2FA() { + return fs.ConfigInput("2fa_do", "config_2fa", "Two-factor authentication: please enter your 2FA code") + } + return nil, nil + case "2fa_do": + code := config.Result + if code == "" { + return fs.ConfigError("authenticate", "2FA codes can't be blank") + } + + icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil) + if err != nil { + return nil, err + } + if err := icloud.SignIn(ctx); err != nil { + return nil, err + } + + if err := icloud.Session.Validate2FACode(ctx, code); err != nil { + return nil, err + } + + m.Set(configTrustToken, icloud.Session.TrustToken) + m.Set(configCookies, icloud.Session.GetCookieString()) + return nil, nil + + case "2fa_error": + if config.Result == "true" { + return fs.ConfigGoto("2fa") + } + return nil, errors.New("2fa authentication failed") + } + return nil, fmt.Errorf("unknown state %q", config.State) +} + +// find item by path. Will not return any children for the item +func (f *Fs) findItem(ctx context.Context, dir string) (item *api.DriveItem, found bool, err error) { + var resp *http.Response + if err = f.pacer.Call(func() (bool, error) { + item, resp, err = f.service.GetItemByPath(ctx, path.Join(f.root, dir)) + return shouldRetry(ctx, resp, err) + }); err != nil { + if item == nil && resp.StatusCode == 404 { + return nil, false, nil + } + return nil, false, err + } + + return item, true, nil +} + +func (f *Fs) findLeafItem(ctx context.Context, pathID string, leaf string) (item *api.DriveItem, found bool, err error) { + items, err := f.listAll(ctx, pathID) + if err != nil { + return nil, false, err + } + for _, item := range items { + if strings.EqualFold(item.FullName(), leaf) { + return item, true, nil + } + } + + return nil, false, nil + +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(ctx context.Context, pathID string, leaf string) (pathIDOut string, found bool, err error) { + item, found, err := f.findLeafItem(ctx, pathID, leaf) + + if err != nil { + return "", found, err + } + + if !found { + return "", false, err + } + + if !item.IsFolder() { + return "", false, fs.ErrorIsFile + } + + return f.IDJoin(item.Drivewsid, item.Etag), true, nil +} + +// Features implements fs.Fs. +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Hashes are not exposed anywhere +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.None) +} + +func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { + root := path.Join(f.root, dir) + if root == "" { + return errors.New("can't purge root directory") + } + + directoryID, etag, err := f.FindDir(ctx, dir, false) + if err != nil { + return err + } + + if check { + item, found, err := f.findItem(ctx, dir) + if err != nil { + return err + } + + if found && item.DirectChildrenCount > 0 { + return fs.ErrorDirectoryNotEmpty + } + } + + var _ *api.DriveItem + var resp *http.Response + if err = f.pacer.Call(func() (bool, error) { + _, resp, err = f.service.MoveItemToTrashByID(ctx, directoryID, etag, true) + return retryResultUnknown(ctx, resp, err) + }); err != nil { + return err + } + + // flush everything from the left of the dir + f.dirCache.FlushDir(dir) + + return nil +} + +// Purge all files in the directory specified +// +// Implement this if you have a way of deleting all the files +// quicker than just running Remove() on the result of List() +// +// Return an error if it doesn't exist +func (f *Fs) Purge(ctx context.Context, dir string) error { + if dir == "" { + return fs.ErrorCantPurge + } + return f.purgeCheck(ctx, dir, false) +} + +func (f *Fs) listAll(ctx context.Context, dirID string) (items []*api.DriveItem, err error) { + var item *api.DriveItem + var resp *http.Response + + if err = f.pacer.Call(func() (bool, error) { + id, _ := f.parseNormalizedID(dirID) + item, resp, err = f.service.GetItemByDriveID(ctx, id, true) + return shouldRetry(ctx, resp, err) + }); err != nil { + return nil, err + } + + items = item.Items + + for i, item := range items { + item.Name = f.opt.Enc.ToStandardName(item.Name) + item.Extension = f.opt.Enc.ToStandardName(item.Extension) + items[i] = item + } + + return items, nil +} + +// List implements fs.Fs. +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + dirRemoteID, err := f.dirCache.FindDir(ctx, dir, false) + if err != nil { + return nil, err + } + + entries = make(fs.DirEntries, 0) + items, err := f.listAll(ctx, dirRemoteID) + + if err != nil { + return nil, err + } + + for _, item := range items { + id := item.Drivewsid + name := item.FullName() + remote := path.Join(dir, name) + if item.IsFolder() { + jid := f.putFolderCache(id, item.Etag, remote) + d := fs.NewDir(remote, item.DateModified).SetID(jid).SetSize(item.AssetQuota) + entries = append(entries, d) + } else { + o, err := f.NewObjectFromDriveItem(ctx, remote, item) + if err != nil { + return nil, err + } + entries = append(entries, o) + } + } + + return entries, nil +} + +// Mkdir implements fs.Fs. +func (f *Fs) Mkdir(ctx context.Context, dir string) error { + _, _, err := f.FindDir(ctx, dir, true) + return err +} + +// Name implements fs.Fs. +func (f *Fs) Name() string { + return f.name +} + +// Precision implements fs.Fs. +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// Copy src to this remote using server-side copy operations. +// +// This is stored with the remote path given. +// +// It returns the destination Object and a possible error. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +// +//nolint:all +func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + // ICloud cooy endpoint is broken. Once they fixed it this can be re-enabled. + return nil, fs.ErrorCantCopy + + // note: so many calls its only just faster then a reupload for big files. + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + + file, pathID, _, err := f.FindPath(ctx, remote, true) + if err != nil { + return nil, err + } + + var resp *http.Response + var info *api.DriveItemRaw + + // make a copy + if err = f.pacer.Call(func() (bool, error) { + info, resp, err = f.service.CopyDocByItemID(ctx, srcObj.itemID) + return retryResultUnknown(ctx, resp, err) + }); err != nil { + return nil, err + } + + // renaming in CopyDocByID endpoint does not work :/ so do it the hard way + + // get new document + var doc *api.Document + if err = f.pacer.Call(func() (bool, error) { + doc, resp, err = f.service.GetDocByItemID(ctx, info.ItemID) + return shouldRetry(ctx, resp, err) + }); err != nil { + return nil, err + } + + // get parentdrive id + var dirDoc *api.Document + if err = f.pacer.Call(func() (bool, error) { + dirDoc, resp, err = f.service.GetDocByItemID(ctx, pathID) + return shouldRetry(ctx, resp, err) + }); err != nil { + return nil, err + } + + // build request + // cant use normal rename as file needs to be "activated" first + + r := api.NewUpdateFileInfo() + r.DocumentID = doc.DocumentID + r.Path.Path = file + r.Path.StartingDocumentID = dirDoc.DocumentID + r.Data.Signature = doc.Data.Signature + r.Data.ReferenceSignature = doc.Data.ReferenceSignature + r.Data.WrappingKey = doc.Data.WrappingKey + r.Data.Size = doc.Data.Size + r.Mtime = srcObj.modTime.UnixMilli() + r.Btime = srcObj.modTime.UnixMilli() + + var item *api.DriveItem + if err = f.pacer.Call(func() (bool, error) { + item, resp, err = f.service.UpdateFile(ctx, &r) + return retryResultUnknown(ctx, resp, err) + }); err != nil { + return nil, err + } + + o, err := f.NewObjectFromDriveItem(ctx, remote, item) + if err != nil { + return nil, err + } + obj := o.(*Object) + + // cheat unit tests + obj.modTime = srcObj.modTime + obj.createdTime = srcObj.createdTime + + return obj, nil +} + +// Put in to the remote path with the modTime given of the given size +// +// When called from outside an Fs by rclone, src.Size() will always be >= 0. +// But for unknown-sized objects (indicated by src.Size() == -1), Put should either +// return an error or upload it properly (rather than e.g. calling panic). +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + size := src.Size() + if size < 0 { + return nil, errors.New("file size unknown") + } + existingObj, err := f.NewObject(ctx, src.Remote()) + switch err { + case nil: + // object is found + return existingObj, existingObj.Update(ctx, in, src, options...) + case fs.ErrorObjectNotFound: + // object not found, so we need to create it + remote := src.Remote() + size := src.Size() + modTime := src.ModTime(ctx) + + obj, err := f.createObject(ctx, remote, modTime, size) + if err != nil { + return nil, err + } + return obj, obj.Update(ctx, in, src, options...) + default: + // real error caught + return nil, err + } +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *Fs) DirCacheFlush() { + f.dirCache.ResetRoot() +} + +// parseNormalizedID parses a normalized ID (may be in the form `driveID#itemID` or just `itemID`) +// and returns itemID, driveID, rootURL. +// Such a normalized ID can come from (*Item).GetID() +// +// Parameters: +// - rid: the normalized ID to be parsed +// +// Returns: +// - id: the itemID extracted from the normalized ID +// - etag: the driveID extracted from the normalized ID, or an empty string if not present +func (f *Fs) parseNormalizedID(rid string) (id string, etag string) { + split := strings.Split(rid, "#") + if len(split) == 1 { + return split[0], "" + } + return split[0], split[1] +} + +// FindPath finds the leaf and directoryID from a normalized path +func (f *Fs) FindPath(ctx context.Context, remote string, create bool) (leaf, directoryID, etag string, err error) { + leaf, jDirectoryID, err := f.dirCache.FindPath(ctx, remote, create) + if err != nil { + return "", "", "", err + } + directoryID, etag = f.parseNormalizedID(jDirectoryID) + return leaf, directoryID, etag, nil +} + +// FindDir finds the directory passed in returning the directory ID +// starting from pathID +func (f *Fs) FindDir(ctx context.Context, path string, create bool) (pathID string, etag string, err error) { + jDirectoryID, err := f.dirCache.FindDir(ctx, path, create) + if err != nil { + return "", "", err + } + directoryID, etag := f.parseNormalizedID(jDirectoryID) + return directoryID, etag, nil +} + +// IDJoin joins the given ID and ETag into a single string with a "#" delimiter. +func (f *Fs) IDJoin(id string, etag string) string { + if strings.Contains(id, "#") { + // already contains an etag, replace + id, _ = f.parseNormalizedID(id) + } + + return strings.Join([]string{id, etag}, "#") +} + +func (f *Fs) putFolderCache(id, etag, remote string) string { + jid := f.IDJoin(id, etag) + f.dirCache.Put(remote, f.IDJoin(id, etag)) + return jid +} + +// Rmdir implements fs.Fs. +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + return f.purgeCheck(ctx, dir, true) +} + +// Root implements fs.Fs. +func (f *Fs) Root() string { + return f.opt.Enc.ToStandardPath(f.root) +} + +// String implements fs.Fs. +func (f *Fs) String() string { + return f.root +} + +// CreateDir makes a directory with pathID as parent and name leaf +// +// This should be implemented by the backend and will be called by the +// dircache package when appropriate. +func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (string, error) { + var item *api.DriveItem + var err error + var found bool + var resp *http.Response + if err = f.pacer.Call(func() (bool, error) { + id, _ := f.parseNormalizedID(pathID) + item, resp, err = f.service.CreateNewFolderByDriveID(ctx, id, f.opt.Enc.FromStandardName(leaf)) + + // check if it went oke + if requestError, ok := err.(*api.RequestError); ok { + if requestError.Status == "unknown" { + fs.Debugf(requestError, " checking if dir is created with separate call.") + time.Sleep(1 * time.Second) // sleep to give icloud time to clear up its mind + item, found, err = f.findLeafItem(ctx, pathID, leaf) + if err != nil { + return false, err + } + + if !found { + // lets assume it failed and retry + return true, err + } + + // success, clear err + err = nil + } + } + + return ignoreResultUnknown(ctx, resp, err) + }); err != nil { + return "", err + } + + return f.IDJoin(item.Drivewsid, item.Etag), err +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server-side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + + srcID, jsrcDirectoryID, srcLeaf, jdstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) + if err != nil { + return err + } + + srcDirectoryID, srcEtag := f.parseNormalizedID(jsrcDirectoryID) + dstDirectoryID, _ := f.parseNormalizedID(jdstDirectoryID) + + _, err = f.move(ctx, srcID, srcDirectoryID, srcLeaf, srcEtag, dstDirectoryID, dstLeaf) + if err != nil { + return err + } + + srcFs.dirCache.FlushDir(srcRemote) + + return nil +} + +func (f *Fs) move(ctx context.Context, ID, srcDirectoryID, srcLeaf, srcEtag, dstDirectoryID, dstLeaf string) (*api.DriveItem, error) { + var resp *http.Response + var item *api.DriveItem + var err error + + // move + if srcDirectoryID != dstDirectoryID { + if err = f.pacer.Call(func() (bool, error) { + id, _ := f.parseNormalizedID(ID) + item, resp, err = f.service.MoveItemByDriveID(ctx, id, srcEtag, dstDirectoryID, true) + return ignoreResultUnknown(ctx, resp, err) + }); err != nil { + return nil, err + } + ID = item.Drivewsid + srcEtag = item.Etag + } + + // rename + if srcLeaf != dstLeaf { + if err = f.pacer.Call(func() (bool, error) { + id, _ := f.parseNormalizedID(ID) + item, resp, err = f.service.RenameItemByDriveID(ctx, id, srcEtag, dstLeaf, true) + return ignoreResultUnknown(ctx, resp, err) + }); err != nil { + return item, err + } + } + + return item, err +} + +// Move moves the src object to the specified remote. +func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + srcLeaf, srcDirectoryID, _, err := srcObj.fs.FindPath(ctx, srcObj.remote, true) + if err != nil { + return nil, err + } + + dstLeaf, dstDirectoryID, _, err := f.FindPath(ctx, remote, true) + if err != nil { + return nil, err + } + + item, err := f.move(ctx, srcObj.driveID, srcDirectoryID, srcLeaf, srcObj.etag, dstDirectoryID, dstLeaf) + if err != nil { + return src, err + } + + return f.NewObjectFromDriveItem(ctx, remote, item) +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Returns the object, leaf, directoryID and error. +// +// Used to create new objects +func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, err error) { + // Create the directory for the object if it doesn't exist + _, _, _, err = f.FindPath(ctx, remote, true) + if err != nil { + return + } + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + modTime: modTime, + size: size, + } + return o, nil +} + +// ReadCookies parses the raw cookie string and returns an array of http.Cookie objects. +func ReadCookies(raw string) []*http.Cookie { + header := http.Header{} + header.Add("Cookie", raw) + request := http.Request{Header: header} + return request.Cookies() +} + +var retryErrorCodes = []int{ + 400, // icloud is a mess, sometimes returns 400 on a perfectly fine request. So just retry + 408, // Request Timeout + 409, // Conflict, retry could fix it. + 429, // Rate exceeded. + 500, // Get occasional 500 Internal Server Error + 502, // Server overload + 503, // Service Unavailable + 504, // Gateway Time-out +} + +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } + + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +func ignoreResultUnknown(ctx context.Context, resp *http.Response, err error) (bool, error) { + if requestError, ok := err.(*api.RequestError); ok { + if requestError.Status == "unknown" { + fs.Debugf(requestError, " ignoring.") + return false, nil + } + } + return shouldRetry(ctx, resp, err) +} + +func retryResultUnknown(ctx context.Context, resp *http.Response, err error) (bool, error) { + if requestError, ok := err.(*api.RequestError); ok { + if requestError.Status == "unknown" { + fs.Debugf(requestError, " retrying.") + return true, err + } + } + return shouldRetry(ctx, resp, err) +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + if opt.Password != "" { + var err error + opt.Password, err = obscure.Reveal(opt.Password) + if err != nil { + return nil, fmt.Errorf("couldn't decrypt user password: %w", err) + } + } + + if opt.TrustToken == "" { + return nil, fmt.Errorf("missing icloud trust token: try refreshing it with \"rclone config reconnect %s:\"", name) + } + + cookies := ReadCookies(opt.Cookies) + + callback := func(session *api.Session) { + m.Set(configCookies, session.GetCookieString()) + } + + icloud, err := api.New( + opt.AppleID, + opt.Password, + opt.TrustToken, + opt.ClientID, + cookies, + callback, + ) + if err != nil { + return nil, err + } + + if err := icloud.Authenticate(ctx); err != nil { + return nil, err + } + + if icloud.Session.Requires2FA() { + return nil, errors.New("trust token expired, please reauth") + } + + root = strings.Trim(root, "/") + + f := &Fs{ + name: name, + root: root, + icloud: icloud, + rootID: "FOLDER::com.apple.CloudDocs::root", + opt: *opt, + pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), + } + f.features = (&fs.Features{ + CanHaveEmptyDirectories: true, + PartialUploads: false, + }).Fill(ctx, f) + + rootID := f.rootID + f.service, err = icloud.DriveService() + if err != nil { + return nil, err + } + + f.dirCache = dircache.New( + root, + rootID, + f, + ) + + err = f.dirCache.FindRoot(ctx, false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + tempF := *f + tempF.dirCache = dircache.New(newRoot, rootID, &tempF) + tempF.root = newRoot + // Make new Fs which is the parent + err = tempF.dirCache.FindRoot(ctx, false) + if err != nil { + // No root so return old f + return f, nil + } + + _, err := tempF.NewObject(ctx, remote) + if err != nil { + if err == fs.ErrorObjectNotFound { + // File doesn't exist so return old f + return f, nil + } + + return nil, err + } + + f.dirCache = tempF.dirCache + f.root = tempF.root + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + + return f, nil +} + +// NewObject creates a new fs.Object from a given remote string. +// +// ctx: The context.Context for the function. +// remote: The remote string representing the object's location. +// Returns an fs.Object and an error. +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + return f.NewObjectFromDriveItem(ctx, remote, nil) +} + +// NewObjectFromDriveItem creates a new fs.Object from a given remote string and DriveItem. +// +// ctx: The context.Context for the function. +// remote: The remote string representing the object's location. +// item: The optional DriveItem to use for initializing the Object. If nil, the function will read the metadata from the remote location. +// Returns an fs.Object and an error. +func (f *Fs) NewObjectFromDriveItem(ctx context.Context, remote string, item *api.DriveItem) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + if item != nil { + err := o.setMetaData(item) + if err != nil { + return nil, err + } + } else { + item, err := f.readMetaData(ctx, remote) + + if err != nil { + return nil, err + } + + err = o.setMetaData(item) + if err != nil { + return nil, err + } + } + + return o, nil +} + +func (f *Fs) readMetaData(ctx context.Context, path string) (item *api.DriveItem, err error) { + leaf, ID, _, err := f.FindPath(ctx, path, false) + + if err != nil { + if err == fs.ErrorDirNotFound { + return nil, fs.ErrorObjectNotFound + } + return nil, err + } + + item, found, err := f.findLeafItem(ctx, ID, leaf) + + if err != nil { + return nil, err + } + + if !found { + return nil, fs.ErrorObjectNotFound + } + + return item, nil +} + +func (o *Object) setMetaData(item *api.DriveItem) (err error) { + if item.IsFolder() { + return fs.ErrorIsDir + } + o.size = item.Size + o.modTime = item.DateModified + o.createdTime = item.DateCreated + o.driveID = item.Drivewsid + o.docID = item.Docwsid + o.itemID = item.Itemid + o.etag = item.Etag + o.downloadURL = item.DownloadURL() + return nil +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.driveID +} + +// Fs implements fs.Object. +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Hash implements fs.Object. +func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// ModTime implements fs.Object. +func (o *Object) ModTime(context.Context) time.Time { + return o.modTime +} + +// Open implements fs.Object. +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { + fs.FixRangeOption(options, o.size) + + // Drive does not support empty files, so we cheat + if o.size == 0 { + return io.NopCloser(bytes.NewBufferString("")), nil + } + + var resp *http.Response + var err error + + if err = o.fs.pacer.Call(func() (bool, error) { + var url string + + //var doc *api.Document + //if o.docID == "" { + //doc, resp, err = o.fs.service.GetDocByItemID(ctx, o.itemID) + //} + + // Can not get the download url on a item to work, so do it the hard way. + url, _, err = o.fs.service.GetDownloadURLByDriveID(ctx, o.driveID) + + resp, err = o.fs.service.DownloadFile(ctx, url, options) + return shouldRetry(ctx, resp, err) + }); err != nil { + return nil, err + } + + return resp.Body, err +} + +// Remote implements fs.Object. +func (o *Object) Remote() string { + return o.remote +} + +// Remove implements fs.Object. +func (o *Object) Remove(ctx context.Context) error { + if o.itemID == "" { + return nil + } + + var resp *http.Response + var err error + if err = o.fs.pacer.Call(func() (bool, error) { + _, resp, err = o.fs.service.MoveItemToTrashByID(ctx, o.driveID, o.etag, true) + return retryResultUnknown(ctx, resp, err) + }); err != nil { + return err + } + + return nil +} + +// SetModTime implements fs.Object. +func (o *Object) SetModTime(ctx context.Context, t time.Time) error { + return fs.ErrorCantSetModTime +} + +// Size implements fs.Object. +func (o *Object) Size() int64 { + return o.size +} + +// Storable implements fs.Object. +func (o *Object) Storable() bool { + return true +} + +// String implements fs.Object. +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Update implements fs.Object. +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + size := src.Size() + if size < 0 { + return errors.New("file size unknown") + } + + remote := o.Remote() + modTime := src.ModTime(ctx) + + leaf, dirID, _, err := o.fs.FindPath(ctx, path.Clean(remote), true) + if err != nil { + return err + } + + // Move current file to trash + if o.driveID != "" { + err = o.Remove(ctx) + if err != nil { + return err + } + } + + name := o.fs.opt.Enc.FromStandardName(leaf) + var resp *http.Response + + // Create document + var uploadInfo *api.UploadResponse + if err = o.fs.pacer.Call(func() (bool, error) { + uploadInfo, resp, err = o.fs.service.CreateUpload(ctx, size, name) + return ignoreResultUnknown(ctx, resp, err) + }); err != nil { + return err + } + + // Upload content + var upload *api.SingleFileResponse + if err = o.fs.pacer.Call(func() (bool, error) { + upload, resp, err = o.fs.service.Upload(ctx, in, size, name, uploadInfo.URL) + return ignoreResultUnknown(ctx, resp, err) + }); err != nil { + return err + } + + //var doc *api.Document + //if err = o.fs.pacer.Call(func() (bool, error) { + // doc, resp, err = o.fs.service.GetDocByItemID(ctx, dirID) + // return ignoreResultUnknown(ctx, resp, err) + //}); err != nil { + // return err + //} + + r := api.NewUpdateFileInfo() + r.DocumentID = uploadInfo.DocumentID + r.Path.Path = name + r.Path.StartingDocumentID = api.GetDocIDFromDriveID(dirID) + //r.Path.StartingDocumentID = doc.DocumentID + r.Data.Receipt = upload.SingleFile.Receipt + r.Data.Signature = upload.SingleFile.Signature + r.Data.ReferenceSignature = upload.SingleFile.ReferenceSignature + r.Data.WrappingKey = upload.SingleFile.WrappingKey + r.Data.Size = upload.SingleFile.Size + r.Mtime = modTime.Unix() * 1000 + r.Btime = modTime.Unix() * 1000 + + // Update metadata + var item *api.DriveItem + if err = o.fs.pacer.Call(func() (bool, error) { + item, resp, err = o.fs.service.UpdateFile(ctx, &r) + return ignoreResultUnknown(ctx, resp, err) + }); err != nil { + return err + } + + err = o.setMetaData(item) + if err != nil { + return err + } + + o.modTime = modTime + o.size = src.Size() + + return nil +} + +// Check interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.Mover = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Object = &Object{} + _ fs.IDer = (*Object)(nil) +) diff --git a/backend/iclouddrive/iclouddrive_test.go b/backend/iclouddrive/iclouddrive_test.go new file mode 100644 index 000000000..13db708ea --- /dev/null +++ b/backend/iclouddrive/iclouddrive_test.go @@ -0,0 +1,18 @@ +//go:build !plan9 && !solaris + +package iclouddrive_test + +import ( + "testing" + + "github.com/rclone/rclone/backend/iclouddrive" + "github.com/rclone/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestICloudDrive:", + NilObject: (*iclouddrive.Object)(nil), + }) +} diff --git a/backend/iclouddrive/iclouddrive_unsupported.go b/backend/iclouddrive/iclouddrive_unsupported.go new file mode 100644 index 000000000..2eeb6b639 --- /dev/null +++ b/backend/iclouddrive/iclouddrive_unsupported.go @@ -0,0 +1,7 @@ +// Build for iclouddrive for unsupported platforms to stop go complaining +// about "no buildable Go source files " + +//go:build plan9 || solaris + +// Package iclouddrive implements the iCloud Drive backend +package iclouddrive \ No newline at end of file diff --git a/bin/make_manual.py b/bin/make_manual.py index c28398dfd..990ab7130 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -52,6 +52,7 @@ docs = [ "hidrive.md", "http.md", "imagekit.md", + "iclouddrive.md", "internetarchive.md", "jottacloud.md", "koofr.md", diff --git a/docs/content/_index.md b/docs/content/_index.md index 2fa03fb06..5bb813a0b 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -132,6 +132,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="Hetzner Storage Box" home="https://www.hetzner.com/storage/storage-box" config="/sftp/#hetzner-storage-box" >}} {{< provider name="HiDrive" home="https://www.strato.de/cloud-speicher/" config="/hidrive/" >}} {{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}} +{{< provider name="iCloud Drive" home="https://icloud.com/" config="/iclouddrive/" >}} {{< provider name="ImageKit" home="https://imagekit.io" config="/imagekit/" >}} {{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}} {{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}} diff --git a/docs/content/docs.md b/docs/content/docs.md index 1d1bdaa76..cb05455b6 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -53,6 +53,7 @@ See the following for detailed instructions for * [Hetzner Storage Box](/sftp/#hetzner-storage-box) * [HiDrive](/hidrive/) * [HTTP](/http/) + * [iCloud Drive](/iclouddrive/) * [Internet Archive](/internetarchive/) * [Jottacloud](/jottacloud/) * [Koofr](/koofr/) diff --git a/docs/content/iclouddrive.md b/docs/content/iclouddrive.md new file mode 100644 index 000000000..e4982cc4b --- /dev/null +++ b/docs/content/iclouddrive.md @@ -0,0 +1,156 @@ +--- +title: "iCloud Drive" +description: "Rclone docs for iCloud Drive" +versionIntroduced: "v1.69" +--- + +# {{< icon "fa fa-cloud" >}} iCloud Drive + + +## Configuration + +The initial setup for an iCloud Drive backend involves getting a trust token/session. +`rclone config` walks you through it. The trust token is valid for 30 days. After which you will have to reauthenticate with rclone reconnect or rclone config. + +Here is an example of how to make a remote called `iclouddrive`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> iclouddrive +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +[snip] +XX / iCloud Drive + \ (iclouddrive) +[snip] +Storage> iclouddrive +Option apple_id. +Apple ID. +Enter a value. +apple_id> APPLEID +Option password. +Password. +Choose an alternative below. +y) Yes, type in my own password +g) Generate random password +y/g> y +Enter the password: +password: +Confirm the password: +password: +Edit advanced config? +y) Yes +n) No (default) +y/n> n +Option config_2fa. +Two-factor authentication: please enter your 2FA code +Enter a value. +config_2fa> 2FACODE +Remote config +-------------------- +[koofr] +- type: iclouddrive +- apple_id: APPLEID +- password: *** ENCRYPTED *** +- cookies: **************************** +- trust_token: **************************** +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +## Advanced Data Protection + +ADP is currently unsupported and need to be disabled + +{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/iclouddrive/iclouddrive.go then run make backenddocs" >}} +### Standard options + +Here are the Standard options specific to iclouddrive (iCloud Drive). + +#### --iclouddrive-apple-id + +Apple ID. + +Properties: + +- Config: apple_id +- Env Var: RCLONE_ICLOUDDRIVE_APPLE_ID +- Type: string +- Required: true + +#### --iclouddrive-password + +Password. + +**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/). + +Properties: + +- Config: password +- Env Var: RCLONE_ICLOUDDRIVE_PASSWORD +- Type: string +- Required: true + +#### --iclouddrive-trust-token + +trust token (internal use) + +Properties: + +- Config: trust_token +- Env Var: RCLONE_ICLOUDDRIVE_TRUST_TOKEN +- Type: string +- Required: false + +#### --iclouddrive-cookies + +cookies (internal use only) + +Properties: + +- Config: cookies +- Env Var: RCLONE_ICLOUDDRIVE_COOKIES +- Type: string +- Required: false + +### Advanced options + +Here are the Advanced options specific to iclouddrive (iCloud Drive). + +#### --iclouddrive-encoding + +The encoding for the backend. + +See the [encoding section in the overview](/overview/#encoding) for more info. + +Properties: + +- Config: encoding +- Env Var: RCLONE_ICLOUDDRIVE_ENCODING +- Type: Encoding +- Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot + +#### --iclouddrive-description + +Description of the remote. + +Properties: + +- Config: description +- Env Var: RCLONE_ICLOUDDRIVE_DESCRIPTION +- Type: string +- Required: false + +{{< rem autogenerated options stop >}} diff --git a/docs/content/overview.md b/docs/content/overview.md index 089889540..6131ff438 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -33,6 +33,7 @@ Here is an overview of the major features of each cloud storage system. | HDFS | - | R/W | No | No | - | - | | HiDrive | HiDrive ¹² | R/W | No | No | - | - | | HTTP | - | R | No | No | R | - | +| iCloud Drive | - | R | No | No | - | - | | Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU | | Jottacloud | MD5 | R/W | Yes | No | R | RW | | Koofr | MD5 | - | Yes | No | - | - | @@ -505,12 +506,13 @@ upon backend-specific capabilities. | Files.com | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | No | Yes | | FTP | No | No | Yes | Yes | No | No | Yes | No | No | No | Yes | | Gofile | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes | -| Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No | No | No | No | +| Google Cloud Storage | Yes | Yes | No | No | No | No | Yes | No | No | No | No | | Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | | Google Photos | No | No | No | No | No | No | No | No | No | No | No | | HDFS | Yes | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes | | HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | No | Yes | | HTTP | No | No | No | No | No | No | No | No | No | No | Yes | +| iCloud Drive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes | | ImageKit | Yes | Yes | Yes | No | No | No | No | No | No | No | Yes | | Internet Archive | No | Yes | No | No | Yes | Yes | No | No | Yes | Yes | No | | Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 6b2938c75..c6fa0db6f 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -75,6 +75,7 @@ HDFS (Hadoop Distributed Filesystem) HiDrive HTTP + iCloud Drive ImageKit Internet Archive Jottacloud diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 7c5ed0feb..3aa68afab 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -503,3 +503,6 @@ backends: - backend: "ulozto" remote: "TestUlozto:" fastlist: false + - backend: "iclouddrive" + remote: "TestICloudDrive:" + fastlist: false