From 2fcb8f5db7c907ad362206fc536577eae6604f12 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 4 Oct 2015 22:08:31 +0100 Subject: [PATCH] Add support for Microsoft One Drive - fixes #10 * Still to do * Copy * Move * MoveDir --- amazonclouddrive/amazonclouddrive.go | 2 +- docs/content/about.md | 1 + docs/content/onedrive.md | 111 ++++ docs/content/overview.md | 1 + docs/layouts/chrome/navbar.html | 1 + fs/operations_test.go | 1 + fstest/fstests/gen_tests.go | 17 +- graphics/rclone-50x50.png | Bin 0 -> 5834 bytes onedrive/api/api.go | 129 ++++ onedrive/api/types.go | 191 ++++++ onedrive/onedrive.go | 852 +++++++++++++++++++++++++++ onedrive/onedrive_test.go | 56 ++ onedrive/replace.go | 91 +++ onedrive/replace_test.go | 30 + rclone.go | 1 + 15 files changed, 1475 insertions(+), 9 deletions(-) create mode 100644 docs/content/onedrive.md create mode 100644 graphics/rclone-50x50.png create mode 100644 onedrive/api/api.go create mode 100644 onedrive/api/types.go create mode 100644 onedrive/onedrive.go create mode 100644 onedrive/onedrive_test.go create mode 100644 onedrive/replace.go create mode 100644 onedrive/replace_test.go diff --git a/amazonclouddrive/amazonclouddrive.go b/amazonclouddrive/amazonclouddrive.go index a4b665336..85edcacd0 100644 --- a/amazonclouddrive/amazonclouddrive.go +++ b/amazonclouddrive/amazonclouddrive.go @@ -479,7 +479,7 @@ func (f *FsAcd) purgeCheck(check bool) error { if check { // check directory is empty empty := true - _, err := f.listAll(rootID, "", false, false, func(node *acd.Node) bool { + _, err = f.listAll(rootID, "", false, false, func(node *acd.Node) bool { switch *node.Kind { case folderKind: empty = false diff --git a/docs/content/about.md b/docs/content/about.md index b0c8b35ad..b45a2c4d9 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -19,6 +19,7 @@ Rclone is a command line program to sync files and directories to and from * Dropbox * Google Cloud Storage * Amazon Cloud Drive + * Microsoft One Drive * The local filesystem Features diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md new file mode 100644 index 000000000..0c6ca7a1a --- /dev/null +++ b/docs/content/onedrive.md @@ -0,0 +1,111 @@ +--- +title: "Microsoft One Drive" +description: "Rclone docs for Microsoft One Drive" +date: "2015-10-14" +--- + + Microsoft One Drive +----------------------------------------- + +Paths are specified as `remote:path` + +Paths may be as deep as required, eg `remote:directory/subdirectory`. + +The initial setup for One Drive involves getting a token from +Microsoft which you need to do in your browser. `rclone config` walks +you through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +n) New remote +d) Delete remote +q) Quit config +e/n/d/q> n +name> remote +What type of source is it? +Choose a number from below + 1) amazon cloud drive + 2) drive + 3) dropbox + 4) google cloud storage + 5) local + 6) onedrive + 7) s3 + 8) swift +type> 6 +Microsoft App Client Id - leave blank normally. +client_id> +Microsoft App Client Secret - leave blank normally. +client_secret> +Remote config +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +client_id = +client_secret = +token = {"access_token":"XXXXXX"} +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Microsoft. This only runs from the moment it +opens your browser to the moment you get back the verification +code. This is on `http://127.0.0.1:53682/` and this it may require +you to unblock it temporarily if you are running a host firewall. + +Once configured you can then use `rclone` like this, + +List directories in top level of your One Drive + + rclone lsd remote: + +List all the files in your One Drive + + rclone ls remote: + +To copy a local directory to an One Drive directory called backup + + rclone copy /home/source remote:backup + +### Modified time and MD5SUMs ### + +One Drive allows modification times to be set on objects accurate to 1 +second. These will be used to detect whether objects need syncing or +not. + +One drive does not support MD5SUMs. This means the `--checksum` flag +will be equivalent to the `--size-only` flag. + +### Deleting files ### + +Any files you delete with rclone will end up in the trash. Microsoft +doesn't provide an API to permanently delete files, nor to empty the +trash, so you will have to do that with one of Microsoft's apps or via +the One Drive website. + +### Limitations ### + +Note that One Drive is case sensitive so you can't have a +file called "Hello.doc" and one called "hello.doc". + +Rclone only supports your default One Drive, and doesn't work with One +Drive for business. Both these issues may be fixed at some point +depending on user demand! + +There are quite a few characters that can't be in One Drive file +names. These can't occur on Windows platforms, but on non-Windows +platforms they are common. Rclone will map these names to and from an +identical looking unicode equivalent. For example if a file has a `?` +in it will be mapped to `？` instead. diff --git a/docs/content/overview.md b/docs/content/overview.md index 4b0761e5d..56db86982 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -23,6 +23,7 @@ Here is an overview of the major features of each cloud storage system. | Dropbox | No | No | Yes | No | | Google Cloud Storage | Yes | Yes | No | No | | Amazon Cloud Drive | Yes | No | Yes | No | +| Microsoft One Drive | No | Yes | Yes | No | | The local filesystem | Yes | Yes | Depends | No | ### MD5SUM ### diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index d134c04c0..d0ae7f562 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -37,6 +37,7 @@
  • Dropbox
  • Google Cloud Storage
  • Amazon Cloud Drive
    • +
  • Microsoft One Drive
  • Local
    • diff --git a/fs/operations_test.go b/fs/operations_test.go index 0f32c31cb..c9d2692df 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -25,6 +25,7 @@ import ( _ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/googlecloudstorage" _ "github.com/ncw/rclone/local" + _ "github.com/ncw/rclone/onedrive" _ "github.com/ncw/rclone/s3" _ "github.com/ncw/rclone/swift" ) diff --git a/fstest/fstests/gen_tests.go b/fstest/fstests/gen_tests.go index ff5a7a24f..de137a4fb 100644 --- a/fstest/fstests/gen_tests.go +++ b/fstest/fstests/gen_tests.go @@ -65,7 +65,7 @@ import ( ) func init() { - fstests.NilObject = fs.Object((*{{ .FsName }}.FsObject{{ .ObjectName }})(nil)) + fstests.NilObject = fs.Object((*{{ .FsName }}.{{ .ObjectName }})(nil)) fstests.RemoteName = "{{ .TestName }}" } @@ -126,12 +126,13 @@ func generateTestProgram(t *template.Template, fns []string, Fsname, ObjectName func main() { fns := findTestFunctions() t := template.Must(template.New("main").Parse(testProgram)) - generateTestProgram(t, fns, "Local", "Local") - generateTestProgram(t, fns, "Swift", "Swift") - generateTestProgram(t, fns, "S3", "S3") - generateTestProgram(t, fns, "Drive", "Drive") - generateTestProgram(t, fns, "GoogleCloudStorage", "Storage") - generateTestProgram(t, fns, "Dropbox", "Dropbox") - generateTestProgram(t, fns, "AmazonCloudDrive", "Acd") + generateTestProgram(t, fns, "Local", "FsObjectLocal") + generateTestProgram(t, fns, "Swift", "FsObjectSwift") + generateTestProgram(t, fns, "S3", "FsObjectS3") + generateTestProgram(t, fns, "Drive", "FsObjectDrive") + generateTestProgram(t, fns, "GoogleCloudStorage", "FsObjectStorage") + generateTestProgram(t, fns, "Dropbox", "FsObjectDropbox") + generateTestProgram(t, fns, "AmazonCloudDrive", "FsObjectAcd") + generateTestProgram(t, fns, "OneDrive", "Object") log.Printf("Done") } diff --git a/graphics/rclone-50x50.png b/graphics/rclone-50x50.png new file mode 100644 index 0000000000000000000000000000000000000000..24cc315d455e05d363eded703ea2da4720c0dd0d GIT binary patch literal 5834 zcmV;*7B%UKP)e zSad^gZEa<4bO1wgWnpw>WFU8GbZ8()Nlj2!fese{02VSyL_t(&-ffzBm=x8y#((Ej zRd>(yY%svc4ub5mZ>|u)a3LzFF)>%7B)MQjqhj2m(WnUqv#Ov(y)lTQ5f@Al5Wxim zS!5480R`Cxh5=@#w_47*f1IlBCeNMcsizydy6b$)`M&pkzaw7!cS?H-a!TWduYm@o zK>(Hj0FC)Sjro6Df+Gc`mLMU)k^-$Yo(4~Wr_6h;K?<-12mvzuElu%ff)Wc7mLM)g z{+WR`>&XO~OaQq+<7u5=npd<~c{D-C{l0Vu6e<}Cv7wLoc& zh!&*GJ$__}HV)AmBnnG_EiDkBl_t;%gbWu18i7Wm0Y?~Pd~N<+{3j8{A)?^Sm;}2j zCddT>=@-7DGhy2jPs>nTo z2#u5iTRNy90N+H!R~R=ag=1N4fA3X3dh#x`QXm9&EY6r;K1`i?c!hN1o}Ty(>1>yE+$WQobe?Xqx=+eR!n<$OIZ!X>zUywnO^#Nw&N=g9FRn z#?Pc^-nBOaranyDE61aVs)NRng1?XNhTCwK;Zq;v(kb_&f!cjLdGnT0_}L6U+th&H zbcWS`e3)cqCEuFzbDFiR!9pXY1k7Fu6!|?WDHM=sk4P0-3}6XCOq$9Ow*(1Gkgy;w zK?VV_n1d}KS9grP@6V<6fJ?CA2{LEuXnW;%$huxwu}qOjpFNHCeXMv1T}OTg0SK+B zY;^(6J9PuVYdB4^q6$BoX5DKqFy{xO_l7_mIxJr}p73n~e z_8}W)Mm8uIkPWX)ppj9j^p(j?kvGO%aZ7OY%g>nq!%+Zi_{&q&ZC{V&Wl6*wdR_D1 zAOySLd>$!4Wz+cSMwZ=r1x-i4qM}`Al;@GDs|`!0pmxVbnogad%VlG5BphD%3Hw(q zLr94oOVF}Me?I)vuXy``A91#^$)voe$OKUY3I=2qWCLA@h^IAvQDqXC9N)h59p+A% zK(65|gTFro<$D}jy^PY*5^UwuZ{iOrzo0!Qx2+{eH;^bRr=+|JfW{*S88qcV+*BhQ z<~>KY;S9&Nt!MGC?*`!8cmE2@cUd=Y7Sgt9+h-6~EJm!NIW2k)VBhlhS$NltG&VKi zYO~;+(&U1o^a?aN1-7TaRiFArdfYsXuA{DD??($cu&Wp~0euG5a_%fH9DErU4Ie?O{uFKd598q4 zPg(QKqa>QOpu>o(4I0oI+p^GFBZPm%T@u{=?~b$!OXQx`MU?FGYw#2POy@Hfb7{-!hF@SS~Fg z`0~BisXuy{Hr@MhV($(D&*j|lBfR&s8>nd8i7~%_gUT+w09f;zJ83s`1l`7c51}U}1itHY;h;+anE$Kk^cZ#p$>uFdr&7eM7>72k<l8l=N3t01cdH5c`!$0Z|ZJbjAk!#cC+ zg9XInak8gtdF7`!(6Vzk+V>eq%o3E9ms4U9UV>6vk_aoGEx@q^mXt^#@{O(}Sr#QCfUudOh-FGNAzV<1lEo#s}y7n6ue0LCjCPm}%!|Y%DCh_nYCHY<_5HIXc z=_cP8;Dya3NH-vJ8ERU$#;vbq{v&s?qkeMju_SMJn>W=Z3u|1eF|6@A$A8MiuNFfk2cQxrIU(=M$q3jqhj_gRT z;f#5(6XV{$ub}6E!Neq_>+2}1sKkoJLT{TW1{9;$5V5L=n}nr3%;lIA*db1m!T?q` z&@{LKDJA&gjhFaz=Ff>&Hs|t3pT|j*@Zp^kDR13@E2sa4T?^+DizoQWoCQb~kZ=sp zl&;daK>&7)or~Te*KmeQ?|6*MranmExvYHSW#0b76U@GAGE)b3VE$7N6AO`YsVzv_ zf}~{_o+AtCC5xD4fx`&%S>(kc6f31fD@E5)*J8!uRQKpdyDP?VXyp=&^;Byc%IbREWWnjM_Qe{=l>_p);dQVv0RJf;vl$5rpq4$kb*gk(Ysk&Oa-8hw& zeTQ-3=!qO#wTO=%zY~BJ&;OR2pZGJDR>Y)@Z5WtF2spZNH7$GgLkhvcb)Nz-_Wq~n zbH!-dweLt-Owzof+|bsr_5~V*7HFmM!ws%v;Q-+nMt89GYF+x5&!|9-t#KWUyLL6b%vH7x9P4ph$dBMMgvzuWi2} zRCnz~=1d*!FC9ZokG?npIt(2__e(}nY6(iM@EKv{vaL|vIJI*#Xic;BU9b}+WX{#& zHZ`(x&Tm;c$2{cF?j3aN(-&W9WJu*yQ3L`_9(YDdhm^+q5ec#-SYb4Lih5?m!=*s^ z0XNNEkvC9=_X(|$L4cMLA&d?XDDZ^Fwx!|8?Ys42;PtmK=H4d}PJ+~#(_|XXg732X z!*}^&-fVhaKFTZ@L!P7v`Fte*s6sTNO&{@UCoH*E#Kpr-Uko#>@IK4234#ELHbp^7 zq!b7QQpoe2;2F4C;BAz5-m@EU{?kTj(V$F{ISBoL*e5TKPJW=Sk*loZ4gzUL9ls2m- zW{doHq8=59!tZ#1HTKSjAD8*i7Fx5$+$qplT8AlNlo1(yPC>s4ikhAsBHV;!xPN4j z9j1rEIKvIV4N0A+3l`ZDa`h*9?6M09(kZH%RgpbghxT0@+a}x8$a^#Hr){tP#LCK1 z8Y3ww3eZ9+MG&Yw?9f`Er9??kQjqg~gku|(Ll-cWC~9(!gm`F-BTdDMA`yfkFrD8R zkWVQc?H9qmFFpZa`QK)-{BN^JRyL!gyqsiJHMvZh69@LvckFeP)wD%xuyv6qwbo{1 zAqg;z^31Wrta;@b4zB%#vSux~cflu!K;cMFp;VTnW zttM*+{YB-=Hf;q0!VIKSGb;SDFSZ3s4@*w(vclZ{6_= z0Iq)Gd3>drGp;vFe(_^AzVQ;GsudlFk7E6sFLAoImaHG->9N+DK^bELH}C%E{C>#$&l~dz;!=+h%qn5}6&%dPc_+hS^^*$<>Va;#uAbYl+agY3wk&|vC zE;NoM{~wfFkTRWc-XSOeNl$5V0c4cM4bBId0>F3C=>}?AwjyZ@j_%mRL)VSM3yED> zRWn}Rc8tbM4&QQc6^Li|rF5W7&+r1xfvp?(>&>J2(b7Fw)vcL3qK#1jwWejC!Bk$* zj(%5vkAYWRONkYdDIxOvL<-|&ClvSWuqhT*c2oN!^+s!Qet?!1-|X7Pu9eF;bL0>; z9Xio%zz_ysF_JRJ;@#Q5XZbsC0Ghj>`73RE^~cqMOvr4-3;R>r3!9p&AeGJW@}x1u zDyzw*8mZg08RdHnxN8PIC;X7A%1X*?!QuyRW#IT57%+AsT5Gf@>=j#tl-7zAlR{UQ zMexE8pX+Pbykaqbd*C)wXY2EO!j8olFk%e%{qY5AYFcx;_BeJ)5<6L5P#Oc{6lsMX zB_}|(L&%aJ*+uh=s6nEMNkOKrmUm}9g74;d_KSVY`*$OE&zg(!eZE}2gqhPOqkWg^ zRy9;qR21eSA*vPvq>RKa3MjVQ_#aT#x&wAek}>nY#A($IUqe$}Eemh`7RR@5KuAeV z_dYE8-NUSU^?A}wjV7vr!VNT$%oG%l&$LEjgltO)ymTX$&>Y^end1ld@zC6bw7j4- zwsIML{Y|{F?GRNhTC!#JXB^nQ1H!^WQ3H?oj0|*P%4CqNLd4<>xcyPukD6$ZtmSa- zz)qG=9m|=o_R_lla1es(EoOThg^bS5a)-iVW`+W7md`|7zMvII3kZCwaWt$FMJYf>P zf^Rl|!G?Ly;WahT@2YDU_v1T|T2p`I5DRBL&EC!HIC1!E+-w$r>Ce4F+fJSN-9K02 z*cK>(_FSZ6qm&|9VFdA3?K>OcIxORPR1h|1JEtOr+kBw4@f==2%o2EK>o~gd0{{m8 z>`B@W{T3Uad6?4XEjhS;C5JYx;g)&JxuEYbHoo;T%b&UzX**c9%~xN3&f!gKxc{ZU zvuXJUEdTplN|I$<{$JNnQdYs@*Z)d97DEa_qBMymB|#9Nf`C8@T+icwXFiS)f{TV< z$+?V+u${a*lM8#4t2A~tFh=se6f+S8>w|@6f(WcNYKpZZEn2+Cjm3Yj?%Vrod zW<0jzu;kSjDXpyHfq9GQGw!>b^AwFfH2RuGUz7Gt`OSq?Kt`kFXol!WqotNuf^tWa zv;|e=WnBH+R~UKs6V&hCO5kSEO40f9v9upP3SVmuuU$cDiwkJgdoaFIXbt5pYOtLc zCk`E?b*C=O`|L}~E6PbGOR*)~JnbH|)~s0YIs`r?B_;G2d<^pza`I5(Oxq)$Q671W) znen&Z$IA!OWVIltML}E-n_$K-jNNi!uaZHL_GE`VA|?fK3ra2WhE&9DN-gMe@c^dI z|Ae8FrV%emqElxnZ(a?+*{}DJtvivQUHogs2e`Qm(zZ!9Z;{7TT7&I4tX%vLnpL;N z&1TuYZZ+S&`6rauv?UinD(m73%M3q*!W1_b6i;>wgIsan)&^~h=x_?*Af_Ou6(4}n z7)TMRC>hs7OB-Lq_C;^9 zVsMKDWfhhD=%L3MFnT=6)*VRsnnq8N3kOhMk)j{*7XG6d3IXE5Z4EjeE(A(tAjYXw>u%_#(>p4Mn(j!1cy zLk_d}CSTL!nPViGK<5wU2tqz1&Oap-34qalwo60FruejtMC4IwUg``N?`iV@!}vrv zjWQarV@pJlh82^RaZnMtiq4k!;U35YX7SNkm{fRVqtKiq!LBfd$RlVG6(zy{0jnBS U0wuL9aR2}S07*qoM6N<$f}BzkNdN!< literal 0 HcmV?d00001 diff --git a/onedrive/api/api.go b/onedrive/api/api.go new file mode 100644 index 000000000..57b74b82e --- /dev/null +++ b/onedrive/api/api.go @@ -0,0 +1,129 @@ +// Package api implements the API for one drive +package api + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/ncw/rclone/fs" +) + +const ( + rootURL = "https://api.onedrive.com/v1.0" // root URL for requests +) + +// Client contains the info to sustain the API +type Client struct { + c *http.Client +} + +// NewClient takes an oauth http.Client and makes a new api instance +func NewClient(c *http.Client) *Client { + return &Client{ + c: c, + } +} + +// Opts contains parameters for Call, CallJSON etc +type Opts struct { + Method string + Path string + Absolute bool // Path is absolute + Body io.Reader + NoResponse bool // set to close Body + ContentType string + ContentLength *int64 + ContentRange string +} + +// checkClose is a utility function used to check the return from +// Close in a defer statement. +func checkClose(c io.Closer, err *error) { + cerr := c.Close() + if *err == nil { + *err = cerr + } +} + +// decodeJSON decodes resp.Body into json +func (api *Client) decodeJSON(resp *http.Response, result interface{}) (err error) { + defer checkClose(resp.Body, &err) + decoder := json.NewDecoder(resp.Body) + return decoder.Decode(result) +} + +// Call makes the call and returns the http.Response +// +// if err != nil then resp.Body will need to be closed +// +// it will return resp if at all possible, even if err is set +func (api *Client) Call(opts *Opts) (resp *http.Response, err error) { + if opts == nil { + return nil, fmt.Errorf("call() called with nil opts") + } + var url string + if opts.Absolute { + url = opts.Path + } else { + url = rootURL + opts.Path + } + req, err := http.NewRequest(opts.Method, url, opts.Body) + if err != nil { + return + } + if opts.ContentType != "" { + req.Header.Add("Content-Type", opts.ContentType) + } + if opts.ContentLength != nil { + req.ContentLength = *opts.ContentLength + } + if opts.ContentRange != "" { + req.Header.Add("Content-Range", opts.ContentRange) + } + req.Header.Add("User-Agent", fs.UserAgent) + resp, err = api.c.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + // Decode error response + errResponse := new(Error) + err = api.decodeJSON(resp, &errResponse) + if err != nil { + return resp, err + } + return resp, errResponse + } + if opts.NoResponse { + return resp, resp.Body.Close() + } + return resp, nil +} + +// CallJSON runs Call and decodes the body as a JSON object into result +// +// If request is not nil then it will be JSON encoded as the body of the request +// +// It will return resp if at all possible, even if err is set +func (api *Client) CallJSON(opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) { + // Set the body up as a JSON object if required + if opts.Body == nil && request != nil { + body, err := json.Marshal(request) + if err != nil { + return nil, err + } + var newOpts = *opts + newOpts.Body = bytes.NewBuffer(body) + newOpts.ContentType = "application/json" + opts = &newOpts + } + resp, err = api.Call(opts) + if err != nil { + return resp, err + } + err = api.decodeJSON(resp, response) + return resp, err +} diff --git a/onedrive/api/types.go b/onedrive/api/types.go new file mode 100644 index 000000000..99cc95d89 --- /dev/null +++ b/onedrive/api/types.go @@ -0,0 +1,191 @@ +// Types passed and returned to and from the API + +package api + +import "time" + +const ( + timeFormat = `"` + time.RFC3339 + `"` +) + +// Error is returned from one drive when things go wrong +type Error struct { + ErrorInfo struct { + Code string `json:"code"` + Message string `json:"message"` + InnerError struct { + Code string `json:"code"` + } `json:"innererror"` + } `json:"error"` +} + +// Error returns a string for the error and statistifes the error interface +func (e *Error) Error() string { + out := e.ErrorInfo.Code + if e.ErrorInfo.InnerError.Code != "" { + out += ": " + e.ErrorInfo.InnerError.Code + } + out += ": " + e.ErrorInfo.Message + return out +} + +// Check Error statisfies the error interface +var _ error = (*Error)(nil) + +// Identity represents an identity of an actor. For example, and actor +// can be a user, device, or application. +type Identity struct { + DisplayName string `json:"displayName"` + ID string `json:"id"` +} + +// IdentitySet is a keyed collection of Identity objects. It is used +// to represent a set of identities associated with various events for +// an item, such as created by or last modified by. +type IdentitySet struct { + User Identity `json:"user"` + Application Identity `json:"application"` + Device Identity `json:"device"` +} + +// Quota groups storage space quota-related information on OneDrive into a single structure. +type Quota struct { + Total int `json:"total"` + Used int `json:"used"` + Remaining int `json:"remaining"` + Deleted int `json:"deleted"` + State string `json:"state"` // normal | nearing | critical | exceeded +} + +// Drive is a representation of a drive resource +type Drive struct { + ID string `json:"id"` + DriveType string `json:"driveType"` + Owner IdentitySet `json:"owner"` + Quota Quota `json:"quota"` +} + +// Timestamp represents represents date and time information for the +// OneDrive API, by using ISO 8601 and is always in UTC time. +type Timestamp time.Time + +// MarshalJSON turns a Timestamp into JSON (in UTC) +func (t *Timestamp) MarshalJSON() (out []byte, err error) { + out = (*time.Time)(t).UTC().AppendFormat(out, timeFormat) + return out, nil +} + +// UnmarshalJSON turns JSON into a Timestamp +func (t *Timestamp) UnmarshalJSON(data []byte) error { + newT, err := time.Parse(timeFormat, string(data)) + if err != nil { + return err + } + *t = Timestamp(newT) + return nil +} + +// ItemReference groups data needed to reference a OneDrive item +// across the service into a single structure. +type ItemReference struct { + DriveID string `json:"driveId"` // Unique identifier for the Drive that contains the item. Read-only. + ID string `json:"id"` // Unique identifier for the item. Read/Write. + Path string `json:"path"` // Path that used to navigate to the item. Read/Write. +} + +// FolderFacet groups folder-related data on OneDrive into a single structure +type FolderFacet struct { + ChildCount int64 `json:"childCount"` // Number of children contained immediately within this container. +} + +// HashesType groups different types of hashes into a single structure, for an item on OneDrive. +type HashesType struct { + Sha1Hash string `json:"sha1Hash"` // base64 encoded SHA1 hash for the contents of the file (if available) + Crc32Hash string `json:"crc32Hash"` // base64 encoded CRC32 value of the file (if available) +} + +// FileFacet groups file-related data on OneDrive into a single structure. +type FileFacet struct { + MimeType string `json:"mimeType"` // The MIME type for the file. This is determined by logic on the server and might not be the value provided when the file was uploaded. + Hashes HashesType `json:"hashes"` // Hashes of the file's binary content, if available. +} + +// FileSystemInfoFacet contains properties that are reported by the +// device's local file system for the local version of an item. This +// facet can be used to specify the last modified date or created date +// of the item as it was on the local device. +type FileSystemInfoFacet struct { + CreatedDateTime Timestamp `json:"createdDateTime"` // The UTC date and time the file was created on a client. + LastModifiedDateTime Timestamp `json:"lastModifiedDateTime"` // The UTC date and time the file was last modified on a client. +} + +// DeletedFacet indicates that the item on OneDrive has been +// deleted. In this version of the API, the presence (non-null) of the +// facet value indicates that the file was deleted. A null (or +// missing) value indicates that the file is not deleted. +type DeletedFacet struct { +} + +// Item represents metadata for an item in OneDrive +type Item struct { + ID string `json:"id"` // The unique identifier of the item within the Drive. Read-only. + Name string `json:"name"` // The name of the item (filename and extension). Read-write. + ETag string `json:"eTag"` // eTag for the entire item (metadata + content). Read-only. + CTag string `json:"cTag"` // An eTag for the content of the item. This eTag is not changed if only the metadata is changed. Read-only. + CreatedBy IdentitySet `json:"createdBy"` // Identity of the user, device, and application which created the item. Read-only. + LastModifiedBy IdentitySet `json:"lastModifiedBy"` // Identity of the user, device, and application which last modified the item. Read-only. + CreatedDateTime Timestamp `json:"createdDateTime"` // Date and time of item creation. Read-only. + LastModifiedDateTime Timestamp `json:"lastModifiedDateTime"` // Date and time the item was last modified. Read-only. + Size int64 `json:"size"` // Size of the item in bytes. Read-only. + ParentReference *ItemReference `json:"parentReference"` // Parent information, if the item has a parent. Read-write. + WebURL string `json:"webUrl"` // URL that displays the resource in the browser. Read-only. + Description string `json:"description"` // Provide a user-visible description of the item. Read-write. + Folder *FolderFacet `json:"folder"` // Folder metadata, if the item is a folder. Read-only. + File *FileFacet `json:"file"` // File metadata, if the item is a file. Read-only. + FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write. + // Image *ImageFacet `json:"image"` // Image metadata, if the item is an image. Read-only. + // Photo *PhotoFacet `json:"photo"` // Photo metadata, if the item is a photo. Read-only. + // Audio *AudioFacet `json:"audio"` // Audio metadata, if the item is an audio file. Read-only. + // Video *VideoFacet `json:"video"` // Video metadata, if the item is a video. Read-only. + // Location *LocationFacet `json:"location"` // Location metadata, if the item has location data. Read-only. + Deleted *DeletedFacet `json:"deleted"` // Information about the deleted state of the item. Read-only. +} + +// ViewDeltaResponse is the response to the view delta method +type ViewDeltaResponse struct { + Value []Item `json:"value"` // An array of Item objects which have been created, modified, or deleted. + NextLink string `json:"@odata.nextLink"` // A URL to retrieve the next available page of changes. + DeltaLink string `json:"@odata.deltaLink"` // A URL returned instead of @odata.nextLink after all current changes have been returned. Used to read the next set of changes in the future. + DeltaToken string `json:"@delta.token"` // A token value that can be used in the query string on manually-crafted calls to view.delta. Not needed if you're using nextLink and deltaLink. +} + +// ListChildrenResponse is the response to the list children method +type ListChildrenResponse struct { + Value []Item `json:"value"` // An array of Item objects + NextLink string `json:"@odata.nextLink"` // A URL to retrieve the next available page of items. +} + +// CreateItemRequest is the request to create an item object +type CreateItemRequest struct { + Name string `json:"name"` // Name of the folder to be created. + Folder FolderFacet `json:"folder"` // Empty Folder facet to indicate that folder is the type of resource to be created. + ConflictBehavior string `json:"@name.conflictBehavior"` // Determines what to do if an item with a matching name already exists in this folder. Accepted values are: rename, replace, and fail (the default). +} + +// SetFileSystemInfo is used to Update an object's FileSystemInfo. +type SetFileSystemInfo struct { + FileSystemInfo FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write. +} + +// CreateUploadResponse is the response from creating an upload session +type CreateUploadResponse struct { + UploadURL string `json:"uploadUrl"` // "https://sn3302.up.1drv.com/up/fe6987415ace7X4e1eF866337", + ExpirationDateTime Timestamp `json:"expirationDateTime"` // "2015-01-29T09:21:55.523Z", + NextExpectedRanges []string `json:"nextExpectedRanges"` // ["0-"] +} + +// UploadFragmentResponse is the response from uploading a fragment +type UploadFragmentResponse struct { + ExpirationDateTime Timestamp `json:"expirationDateTime"` // "2015-01-29T09:21:55.523Z", + NextExpectedRanges []string `json:"nextExpectedRanges"` // ["0-"] +} diff --git a/onedrive/onedrive.go b/onedrive/onedrive.go new file mode 100644 index 000000000..e9de87b78 --- /dev/null +++ b/onedrive/onedrive.go @@ -0,0 +1,852 @@ +// Package onedrive provides an interface to the Microsoft One Drive +// object storage system. +package onedrive + +import ( + "bytes" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strings" + "sync" + "time" + + "github.com/ncw/rclone/dircache" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/oauthutil" + "github.com/ncw/rclone/onedrive/api" + "github.com/ncw/rclone/pacer" + "github.com/spf13/pflag" + "golang.org/x/oauth2" +) + +const ( + rcloneClientID = "0000000044165769" + rcloneClientSecret = "0+be4+jYw+7018HY6P3t/Izo+pTc+Yvt8+fy8NHU094=" + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential +) + +// Globals +var ( + // Description of how to auth for this app + oauthConfig = &oauth2.Config{ + Scopes: []string{ + "wl.signin", // Allow single sign-on capabilities + "wl.offline_access", // Allow receiving a refresh token + "onedrive.readwrite", // r/w perms to all of a user's OneDrive files + }, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://login.live.com/oauth20_authorize.srf", + TokenURL: "https://login.live.com/oauth20_token.srf", + }, + ClientID: rcloneClientID, + ClientSecret: fs.Reveal(rcloneClientSecret), + RedirectURL: oauthutil.RedirectPublicURL, + } + chunkSize = fs.SizeSuffix(10 * 1024 * 1024) + uploadCutoff = fs.SizeSuffix(10 * 1024 * 1024) +) + +// Register with Fs +func init() { + fs.Register(&fs.Info{ + Name: "onedrive", + NewFs: NewFs, + Config: func(name string) { + err := oauthutil.Config(name, oauthConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + }, + Options: []fs.Option{{ + Name: oauthutil.ConfigClientID, + Help: "Microsoft App Client Id - leave blank normally.", + }, { + Name: oauthutil.ConfigClientSecret, + Help: "Microsoft App Client Secret - leave blank normally.", + }}, + }) + pflag.VarP(&chunkSize, "onedrive-chunk-size", "", "Above this size files will be chunked - must be multiple of 320k.") + pflag.VarP(&uploadCutoff, "onedrive-upload-cutoff", "", "Cutoff for switching to chunked upload - must be <= 100MB") +} + +// Fs represents a remote one drive +type Fs struct { + name string // name of this remote + srv *api.Client // the connection to the one drive server + root string // the path we are working on + dirCache *dircache.DirCache // Map of directory path to directory id + pacer *pacer.Pacer // pacer for API calls +} + +// Object describes a one drive object +// +// Will definitely have info but maybe not meta +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + hasMetaData bool // whether info below has been set + size int64 // size of the object + modTime time.Time // modification time of the object + id string // ID of the object +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("One drive root '%s'", f.root) +} + +// Pattern to match a one drive path +var matcher = regexp.MustCompile(`^([^/]*)(.*)$`) + +// parsePath parses an one drive 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(resp *http.Response, err error) (bool, error) { + return fs.ShouldRetry(err) || fs.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// readMetaDataForPath reads the metadata from the path +func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Response, err error) { + opts := api.Opts{ + Method: "GET", + Path: "/drive/root:/" + replaceReservedChars(path), + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &info) + return shouldRetry(resp, err) + }) + return info, resp, err +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string) (fs.Fs, error) { + root = parsePath(root) + oAuthClient, err := oauthutil.NewClient(name, oauthConfig) + if err != nil { + log.Fatalf("Failed to configure One Drive: %v", err) + } + + f := &Fs{ + name: name, + root: root, + srv: api.NewClient(oAuthClient), + pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant), + } + + // Get rootID + rootInfo, _, err := f.readMetaDataForPath("") + if err != nil || rootInfo.ID == "" { + return nil, fmt.Errorf("Failed to get root: %v", err) + } + + f.dirCache = dircache.New(root, rootInfo.ID, f) + + // Find the current root + err = f.dirCache.FindRoot(false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + newF := *f + newF.dirCache = dircache.New(newRoot, rootInfo.ID, &newF) + newF.root = newRoot + // Make new Fs which is the parent + err = newF.dirCache.FindRoot(false) + if err != nil { + // No root so return old f + return f, nil + } + obj := newF.newObjectWithInfo(remote, nil) + if obj == nil { + // File doesn't exist so return old f + return f, nil + } + // return a Fs Limited to this object + return fs.NewLimited(&newF, obj), nil + } + return f, nil +} + +// rootSlash returns root with a slash on if it is empty, otherwise empty string +func (f *Fs) rootSlash() string { + if f.root == "" { + return f.root + } + return f.root + "/" +} + +// Return an Object from a path +// +// May return nil if an error occurred +func (f *Fs) newObjectWithInfo(remote string, info *api.Item) fs.Object { + o := &Object{ + fs: f, + remote: remote, + } + if info != nil { + // Set info + o.setMetaData(info) + } else { + err := o.readMetaData() // reads info and meta, returning an error + if err != nil { + // logged already FsDebug("Failed to read info: %s", err) + return nil + } + } + return o +} + +// NewFsObject returns an Object from a path +// +// May return nil if an error occurred +func (f *Fs) NewFsObject(remote string) fs.Object { + return f.newObjectWithInfo(remote, nil) +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) { + // fs.Debug(f, "FindLeaf(%q, %q)", pathID, leaf) + parent, ok := f.dirCache.GetInv(pathID) + if !ok { + return "", false, fmt.Errorf("Couldn't find parent ID") + } + path := leaf + if parent != "" { + path = parent + "/" + path + } + if f.dirCache.FoundRoot() { + path = f.rootSlash() + path + } + info, resp, err := f.readMetaDataForPath(path) + if err != nil { + if resp != nil && resp.StatusCode == http.StatusNotFound { + return "", false, nil + } + return "", false, err + } + if info.Folder == nil { + return "", false, fmt.Errorf("Found file when looking for folder") + } + return info.ID, true, nil +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) { + // fs.Debug(f, "CreateDir(%q, %q)\n", pathID, leaf) + var resp *http.Response + var info *api.Item + opts := api.Opts{ + Method: "POST", + Path: "/drive/items/" + pathID + "/children", + } + mkdir := api.CreateItemRequest{ + Name: leaf, + ConflictBehavior: "fail", + } + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, &mkdir, &info) + return shouldRetry(resp, err) + }) + if err != nil { + //fmt.Printf("...Error %v\n", err) + return "", err + } + //fmt.Printf("...Id %q\n", *info.Id) + return info.ID, nil +} + +// list the objects into the function supplied +// +// If directories is set it only sends directories +// User function to process a File item from listAll +// +// Should return true to finish processing +type listAllFn func(*api.Item) bool + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { + // Top parameter asks for bigger pages of data + // https://dev.onedrive.com/odata/optional-query-parameters.htm + opts := api.Opts{ + Method: "GET", + Path: "/drive/items/" + dirID + "/children?top=1000", + } +OUTER: + for { + var result api.ListChildrenResponse + var resp *http.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(&opts, nil, &result) + return shouldRetry(resp, err) + }) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "Couldn't list files: %v", err) + break + } + if len(result.Value) == 0 { + break + } + for i := range result.Value { + item := &result.Value[i] + isFolder := item.Folder != nil + if isFolder { + if filesOnly { + continue + } + } else { + if directoriesOnly { + continue + } + } + if item.Deleted != nil { + continue + } + item.Name = restoreReservedChars(item.Name) + if fn(item) { + found = true + break OUTER + } + } + if result.NextLink == "" { + break + } + opts.Path = result.NextLink + opts.Absolute = true + } + return +} + +// Path should be directory path either "" or "path/" +// +// List the directory using a recursive list from the root +// +// This fetches the minimum amount of stuff but does more API calls +// which makes it slow +func (f *Fs) listDirRecursive(dirID string, path string, out fs.ObjectsChan) error { + var subError error + // Make the API request + var wg sync.WaitGroup + _, err := f.listAll(dirID, false, false, func(info *api.Item) bool { + // Recurse on directories + if info.Folder != nil { + wg.Add(1) + folder := path + info.Name + "/" + fs.Debug(f, "Reading %s", folder) + go func() { + defer wg.Done() + err := f.listDirRecursive(info.ID, folder, out) + if err != nil { + subError = err + fs.ErrorLog(f, "Error reading %s:%s", folder, err) + } + }() + } else { + if fs := f.newObjectWithInfo(path+info.Name, info); fs != nil { + out <- fs + } + } + return false + }) + wg.Wait() + fs.Debug(f, "Finished reading %s", path) + if err != nil { + return err + } + if subError != nil { + return subError + } + return nil +} + +// List walks the path returning a channel of Objects +func (f *Fs) List() fs.ObjectsChan { + out := make(fs.ObjectsChan, fs.Config.Checkers) + go func() { + defer close(out) + err := f.dirCache.FindRoot(false) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "Couldn't find root: %s", err) + } else { + err = f.listDirRecursive(f.dirCache.RootID(), "", out) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "List failed: %s", err) + } + } + }() + return out +} + +// ListDir lists the directories +func (f *Fs) ListDir() fs.DirChan { + out := make(fs.DirChan, fs.Config.Checkers) + go func() { + defer close(out) + err := f.dirCache.FindRoot(false) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "Couldn't find root: %s", err) + } else { + _, err := f.listAll(f.dirCache.RootID(), true, false, func(item *api.Item) bool { + dir := &fs.Dir{ + Name: item.Name, + Bytes: -1, + Count: -1, + When: time.Time(item.LastModifiedDateTime), + } + if item.Folder != nil { + dir.Count = item.Folder.ChildCount + } + out <- dir + return false + }) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "ListDir failed: %s", err) + } + } + }() + return out +} + +// Put the object into the container +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { + // Create the directory for the object if it doesn't exist + _, _, err := f.dirCache.FindPath(remote, true) + if err != nil { + return nil, err + } + // Temporary Object under construction + o := &Object{ + fs: f, + remote: remote, + } + return o, o.Update(in, modTime, size) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir() error { + return f.dirCache.FindRoot(true) +} + +// deleteObject removes an object by ID +func (f *Fs) deleteObject(id string) error { + opts := api.Opts{ + Method: "DELETE", + Path: "/drive/items/" + id, + NoResponse: true, + } + return f.pacer.Call(func() (bool, error) { + resp, err := f.srv.Call(&opts) + return shouldRetry(resp, err) + }) +} + +// purgeCheck removes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(check bool) error { + if f.root == "" { + return fmt.Errorf("Can't purge root directory") + } + dc := f.dirCache + err := dc.FindRoot(false) + if err != nil { + return err + } + rootID := dc.RootID() + item, _, err := f.readMetaDataForPath(f.root) + if err != nil { + return err + } + if item.Folder == nil { + return fmt.Errorf("Not a folder") + } + if check && item.Folder.ChildCount != 0 { + return fmt.Errorf("Folder not empty") + } + err = f.deleteObject(rootID) + if err != nil { + return err + } + f.dirCache.ResetRoot() + if err != nil { + return err + } + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir() error { + return f.purgeCheck(true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// Copy src to this remote using server side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +//func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { +// srcObj, ok := src.(*Object) +// if !ok { +// fs.Debug(src, "Can't copy - not same remote type") +// return nil, fs.ErrorCantCopy +// } +// srcFs := srcObj.acd +// _, err := f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil) +// if err != nil { +// return nil, err +// } +// return f.NewFsObject(remote), nil +//} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + return f.purgeCheck(false) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Fs { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// srvPath returns a path for use in server +func (o *Object) srvPath() string { + return replaceReservedChars(o.fs.rootSlash() + o.remote) +} + +// Md5sum returns the Md5sum of an object returning a lowercase hex string +func (o *Object) Md5sum() (string, error) { + return "", nil // not supported by one drive +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + err := o.readMetaData() + if err != nil { + fs.Log(o, "Failed to read metadata: %s", err) + return 0 + } + return o.size +} + +// setMetaData sets the metadata from info +func (o *Object) setMetaData(info *api.Item) { + o.hasMetaData = true + o.size = info.Size + if info.FileSystemInfo != nil { + o.modTime = time.Time(info.FileSystemInfo.LastModifiedDateTime) + } else { + o.modTime = time.Time(info.LastModifiedDateTime) + } + o.id = info.ID +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData() (err error) { + if o.hasMetaData { + return nil + } + // leaf, directoryID, err := o.fs.dirCache.FindPath(o.remote, false) + // if err != nil { + // return err + // } + info, _, err := o.fs.readMetaDataForPath(o.srvPath()) + if err != nil { + fs.Debug(o, "Failed to read info: %s", err) + return err + } + o.setMetaData(info) + return nil +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Log(o, "Failed to read metadata: %s", err) + return time.Now() + } + return o.modTime +} + +// setModTime sets the modification time of the local fs object +func (o *Object) setModTime(modTime time.Time) (*api.Item, error) { + opts := api.Opts{ + Method: "PATCH", + Path: "/drive/root:/" + o.srvPath(), + } + update := api.SetFileSystemInfo{ + FileSystemInfo: api.FileSystemInfoFacet{ + CreatedDateTime: api.Timestamp(modTime), + LastModifiedDateTime: api.Timestamp(modTime), + }, + } + var info *api.Item + err := o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.CallJSON(&opts, &update, &info) + return shouldRetry(resp, err) + }) + return info, err +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(modTime time.Time) { + info, err := o.setModTime(modTime) + if err != nil { + fs.Stats.Error() + fs.ErrorLog(o, "Failed to update remote mtime: %v", err) + } + o.setMetaData(info) +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open() (in io.ReadCloser, err error) { + if o.id == "" { + return nil, fmt.Errorf("Can't download no id") + } + var resp *http.Response + opts := api.Opts{ + Method: "GET", + Path: "/drive/items/" + o.id + "/content", + } + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return shouldRetry(resp, err) + }) + if err != nil { + return nil, err + } + return resp.Body, err +} + +// createUploadSession creates an upload session for the object +func (o *Object) createUploadSession() (response *api.CreateUploadResponse, err error) { + opts := api.Opts{ + Method: "POST", + Path: "/drive/root:/" + o.srvPath() + ":/upload.createSession", + } + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(&opts, nil, &response) + return shouldRetry(resp, err) + }) + return +} + +// uploadFragment uploads a part +func (o *Object) uploadFragment(url string, start int64, totalSize int64, buf []byte) (err error) { + bufSize := int64(len(buf)) + opts := api.Opts{ + Method: "PUT", + Path: url, + Absolute: true, + ContentLength: &bufSize, + ContentRange: fmt.Sprintf("bytes %d-%d/%d", start, start+bufSize-1, totalSize), + Body: bytes.NewReader(buf), + } + var response api.UploadFragmentResponse + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(&opts, nil, &response) + return shouldRetry(resp, err) + }) + return err +} + +// cancelUploadSession cancels an upload session +func (o *Object) cancelUploadSession(url string) (err error) { + opts := api.Opts{ + Method: "DELETE", + Path: url, + Absolute: true, + NoResponse: true, + } + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(&opts) + return shouldRetry(resp, err) + }) + return +} + +// uploadMultipart uploads a file using multipart upload +func (o *Object) uploadMultipart(in io.Reader, size int64) (err error) { + if chunkSize%(320*1024) != 0 { + return fmt.Errorf("Chunk size %d is not a multiple of 320k", chunkSize) + } + + // Create upload session + fs.Debug(o, "Starting multipart upload") + session, err := o.createUploadSession() + if err != nil { + return err + } + uploadURL := session.UploadURL + + // Cancel the session if something went wrong + defer func() { + if err != nil { + fs.Debug(o, "Cancelling multipart upload") + cancelErr := o.cancelUploadSession(uploadURL) + if cancelErr != nil { + fs.Log(o, "Failed to cancel multipart upload: %v", err) + } + } + }() + + // Upload the chunks + remaining := size + position := int64(0) + buf := make([]byte, int64(chunkSize)) + for remaining > 0 { + n := int64(chunkSize) + if remaining < n { + n = remaining + buf = buf[:n] + } + _, err = io.ReadFull(in, buf) + if err != nil { + return err + } + fs.Debug(o, "Uploading segment %d/%d size %d", position, size, n) + err = o.uploadFragment(uploadURL, position, size, buf) + if err != nil { + return err + } + remaining -= n + position += n + } + + return nil +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, modTime time.Time, size int64) (err error) { + var info *api.Item + if size <= int64(uploadCutoff) { + // This is for less than 100 MB of content + var resp *http.Response + opts := api.Opts{ + Method: "PUT", + Path: "/drive/root:/" + o.srvPath() + ":/content", + Body: in, + } + err = o.fs.pacer.CallNoRetry(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(&opts, nil, &info) + return shouldRetry(resp, err) + }) + if err != nil { + return err + } + o.setMetaData(info) + } else { + err = o.uploadMultipart(in, size) + if err != nil { + return err + } + } + // Set the mod time now and read metadata + info, err = o.setModTime(modTime) + if err != nil { + return err + } + o.setMetaData(info) + return nil +} + +// Remove an object +func (o *Object) Remove() error { + return o.fs.deleteObject(o.id) +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + // _ fs.Copier = (*Fs)(nil) + // _ fs.Mover = (*Fs)(nil) + // _ fs.DirMover = (*Fs)(nil) + _ fs.Object = (*Object)(nil) +) diff --git a/onedrive/onedrive_test.go b/onedrive/onedrive_test.go new file mode 100644 index 000000000..392997367 --- /dev/null +++ b/onedrive/onedrive_test.go @@ -0,0 +1,56 @@ +// Test OneDrive filesystem interface +// +// Automatically generated - DO NOT EDIT +// Regenerate with: make gen_tests +package onedrive_test + +import ( + "testing" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" + "github.com/ncw/rclone/onedrive" +) + +func init() { + fstests.NilObject = fs.Object((*onedrive.Object)(nil)) + fstests.RemoteName = "TestOneDrive:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) } +func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } diff --git a/onedrive/replace.go b/onedrive/replace.go new file mode 100644 index 000000000..1d38d56df --- /dev/null +++ b/onedrive/replace.go @@ -0,0 +1,91 @@ +/* +Translate file names for one drive + +OneDrive reserved characters + +The following characters are OneDrive reserved characters, and can't +be used in OneDrive folder and file names. + + onedrive-reserved = "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|" + onedrive-business-reserved + = "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|" / "#" / "%" + +Note: Folder names can't end with a period (.). + +Note: OneDrive for Business file or folder names cannot begin with a +tilde ('~'). + +*/ + +package onedrive + +import ( + "regexp" + "strings" +) + +// charMap holds replacements for characters +// +// Onedrive has a restricted set of characters compared to other cloud +// storage systems, so we to map these to the FULLWIDTH unicode +// equivalents +// +// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS +var ( + charMap = map[rune]rune{ + '\\': '＼', // FULLWIDTH REVERSE SOLIDUS + '*': '＊', // FULLWIDTH ASTERISK + '<': '＜', // FULLWIDTH LESS-THAN SIGN + '>': '＞', // FULLWIDTH GREATER-THAN SIGN + '?': '？', // FULLWIDTH QUESTION MARK + ':': '：', // FULLWIDTH COLON + '|': '｜', // FULLWIDTH VERTICAL LINE + '#': '＃', // FULLWIDTH NUMBER SIGN + '%': '％', // FULLWIDTH PERCENT SIGN + '"': '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved + '.': '．', // FULLWIDTH FULL STOP + '~': '～', // FULLWIDTH TILDE + ' ': '␠', // SYMBOL FOR SPACE + } + invCharMap map[rune]rune + fixEndingInPeriod = regexp.MustCompile(`\.(/|$)`) + fixStartingWithTilde = regexp.MustCompile(`(/|^)~`) + fixStartingWithSpace = regexp.MustCompile(`(/|^) `) +) + +func init() { + // Create inverse charMap + invCharMap = make(map[rune]rune, len(charMap)) + for k, v := range charMap { + invCharMap[v] = k + } +} + +// replaceReservedChars takes a path and substitutes any reserved +// characters in it +func replaceReservedChars(in string) string { + // Folder names can't end with a period '.' + in = fixEndingInPeriod.ReplaceAllString(in, string(charMap['.'])+"$1") + // OneDrive for Business file or folder names cannot begin with a tilde '~' + in = fixStartingWithTilde.ReplaceAllString(in, "$1"+string(charMap['~'])) + // Apparently file names can't start with space either + in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' '])) + // Replace reserved characters + return strings.Map(func(c rune) rune { + if replacement, ok := charMap[c]; ok && c != '.' && c != '~' && c != ' ' { + return replacement + } + return c + }, in) +} + +// restoreReservedChars takes a path and undoes any substitutions +// made by replaceReservedChars +func restoreReservedChars(in string) string { + return strings.Map(func(c rune) rune { + if replacement, ok := invCharMap[c]; ok { + return replacement + } + return c + }, in) +} diff --git a/onedrive/replace_test.go b/onedrive/replace_test.go new file mode 100644 index 000000000..bac8a590e --- /dev/null +++ b/onedrive/replace_test.go @@ -0,0 +1,30 @@ +package onedrive + +import "testing" + +func TestReplace(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"", ""}, + {"abc 123", "abc 123"}, + {`\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~`}, + {`\*<>?:|#%".~/\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~/＼＊＜＞？：｜＃％＂.~`}, + {" leading space", "␠leading space"}, + {"~leading tilde", "～leading tilde"}, + {"trailing dot.", "trailing dot．"}, + {" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"}, + {"~leading tilde/~leading tilde/~leading tilde", "～leading tilde/～leading tilde/～leading tilde"}, + {"trailing dot./trailing dot./trailing dot.", "trailing dot．/trailing dot．/trailing dot．"}, + } { + got := replaceReservedChars(test.in) + if got != test.out { + t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got) + } + got2 := restoreReservedChars(got) + if got2 != test.in { + t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2) + } + } +} diff --git a/rclone.go b/rclone.go index c241227f0..ae6784020 100644 --- a/rclone.go +++ b/rclone.go @@ -21,6 +21,7 @@ import ( _ "github.com/ncw/rclone/dropbox" _ "github.com/ncw/rclone/googlecloudstorage" _ "github.com/ncw/rclone/local" + _ "github.com/ncw/rclone/onedrive" _ "github.com/ncw/rclone/s3" _ "github.com/ncw/rclone/swift" )