diff --git a/fs/operations_test.go b/fs/operations_test.go index ea8c03b25..6b3175b59 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -30,6 +30,7 @@ import ( _ "github.com/ncw/rclone/onedrive" _ "github.com/ncw/rclone/s3" _ "github.com/ncw/rclone/swift" + _ "github.com/ncw/rclone/yandex" ) // Globals diff --git a/fs/test_all.go b/fs/test_all.go index bda895400..47d40c98e 100644 --- a/fs/test_all.go +++ b/fs/test_all.go @@ -26,6 +26,7 @@ var ( "TestOneDrive:", "TestHubic:", "TestB2:", + "TestYandex:", } binary = "fs.test" // Flags diff --git a/fstest/fstests/gen_tests.go b/fstest/fstests/gen_tests.go index b5732bb7a..040e94ed8 100644 --- a/fstest/fstests/gen_tests.go +++ b/fstest/fstests/gen_tests.go @@ -134,5 +134,6 @@ func main() { generateTestProgram(t, fns, "OneDrive") generateTestProgram(t, fns, "Hubic") generateTestProgram(t, fns, "B2") + generateTestProgram(t, fns, "Yandex") log.Printf("Done") } diff --git a/rclone.go b/rclone.go index f6c201211..3149eb77c 100644 --- a/rclone.go +++ b/rclone.go @@ -26,6 +26,7 @@ import ( _ "github.com/ncw/rclone/onedrive" _ "github.com/ncw/rclone/s3" _ "github.com/ncw/rclone/swift" + _ "github.com/ncw/rclone/yandex" ) // Globals diff --git a/yandex/api/api_request.go b/yandex/api/api_request.go new file mode 100644 index 000000000..5ce3d6b55 --- /dev/null +++ b/yandex/api/api_request.go @@ -0,0 +1,5 @@ +package src + +type apiRequest interface { + Request() *HTTPRequest +} diff --git a/yandex/api/api_upload.go b/yandex/api/api_upload.go new file mode 100644 index 000000000..8db441373 --- /dev/null +++ b/yandex/api/api_upload.go @@ -0,0 +1,34 @@ +package src + +//from yadisk + +import ( + "io" + "net/http" +) + +//RootAddr is the base URL for Yandex Disk API. +const RootAddr = "https://cloud-api.yandex.com" //also https://cloud-api.yandex.net and https://cloud-api.yandex.ru + +func (c *Client) setRequestScope(req *http.Request) { + req.Header.Add("Accept", "application/json") + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", "OAuth "+c.token) +} + +func (c *Client) scopedRequest(method, urlPath string, body io.Reader) (*http.Request, error) { + fullURL := RootAddr + if urlPath[:1] != "/" { + fullURL += "/" + urlPath + } else { + fullURL += urlPath + } + + req, err := http.NewRequest(method, fullURL, body) + if err != nil { + return req, err + } + + c.setRequestScope(req) + return req, nil +} diff --git a/yandex/api/client.go b/yandex/api/client.go new file mode 100644 index 000000000..a2d727b3b --- /dev/null +++ b/yandex/api/client.go @@ -0,0 +1,134 @@ +package src + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strings" +) + +//Client struct +type Client struct { + token string + basePath string + HTTPClient *http.Client +} + +//NewClient creates new client +func NewClient(token string, client ...*http.Client) *Client { + return newClientInternal( + token, + "https://cloud-api.yandex.com/v1/disk", //also "https://cloud-api.yandex.net/v1/disk" "https://cloud-api.yandex.ru/v1/disk" + client...) +} + +func newClientInternal(token string, basePath string, client ...*http.Client) *Client { + c := &Client{ + token: token, + basePath: basePath, + } + if len(client) != 0 { + c.HTTPClient = client[0] + } else { + c.HTTPClient = http.DefaultClient + } + return c +} + +//ErrorHandler type +type ErrorHandler func(*http.Response) error + +var defaultErrorHandler ErrorHandler = func(resp *http.Response) error { + if resp.StatusCode/100 == 5 { + return errors.New("server error") + } + + if resp.StatusCode/100 == 4 { + var response DiskClientError + contents, _ := ioutil.ReadAll(resp.Body) + err := json.Unmarshal(contents, &response) + if err != nil { + return err + } + return response + } + + if resp.StatusCode/100 == 3 { + return errors.New("redirect error") + } + return nil +} + +func (HTTPRequest *HTTPRequest) run(client *Client) ([]byte, error) { + var err error + values := make(url.Values) + if HTTPRequest.Parameters != nil { + for k, v := range HTTPRequest.Parameters { + values.Set(k, fmt.Sprintf("%v", v)) + } + } + + var req *http.Request + if HTTPRequest.Method == "POST" { + // TODO json serialize + req, err = http.NewRequest( + "POST", + client.basePath+HTTPRequest.Path, + strings.NewReader(values.Encode())) + if err != nil { + return nil, err + } + // TODO + // req.Header.Set("Content-Type", "application/json") + } else { + req, err = http.NewRequest( + HTTPRequest.Method, + client.basePath+HTTPRequest.Path+"?"+values.Encode(), + nil) + if err != nil { + return nil, err + } + } + + for headerName := range HTTPRequest.Headers { + var headerValues = HTTPRequest.Headers[headerName] + for _, headerValue := range headerValues { + req.Header.Set(headerName, headerValue) + } + } + return runRequest(client, req) +} + +func runRequest(client *Client, req *http.Request) ([]byte, error) { + return runRequestWithErrorHandler(client, req, defaultErrorHandler) +} + +func runRequestWithErrorHandler(client *Client, req *http.Request, errorHandler ErrorHandler) ([]byte, error) { + resp, err := client.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer CheckClose(resp.Body, &err) + + return checkResponseForErrorsWithErrorHandler(resp, errorHandler) +} + +func checkResponseForErrorsWithErrorHandler(resp *http.Response, errorHandler ErrorHandler) ([]byte, error) { + if resp.StatusCode/100 > 2 { + return nil, errorHandler(resp) + } + return ioutil.ReadAll(resp.Body) +} + +// CheckClose is a utility function used to check the return from +// Close in a defer statement. +func CheckClose(c io.Closer, err *error) { + cerr := c.Close() + if *err == nil { + *err = cerr + } +} diff --git a/yandex/api/custom_property.go b/yandex/api/custom_property.go new file mode 100644 index 000000000..f9b28fee4 --- /dev/null +++ b/yandex/api/custom_property.go @@ -0,0 +1,50 @@ +package src + +import ( + "bytes" + "encoding/json" + "io" + "net/url" +) + +//CustomPropertyResponse struct we send and is returned by the API for CustomProperty request. +type CustomPropertyResponse struct { + CustomProperties map[string]interface{} `json:"custom_properties"` +} + +//SetCustomProperty will set specified data from Yandex Disk +func (c *Client) SetCustomProperty(remotePath string, property string, value string) error { + rcm := map[string]interface{}{ + property: value, + } + cpr := CustomPropertyResponse{rcm} + data, _ := json.Marshal(cpr) + body := bytes.NewReader(data) + err := c.SetCustomPropertyRequest(remotePath, body) + if err != nil { + return err + } + return err +} + +//SetCustomPropertyRequest will make an CustomProperty request and return a URL to CustomProperty data to. +func (c *Client) SetCustomPropertyRequest(remotePath string, body io.Reader) error { + values := url.Values{} + values.Add("path", remotePath) + req, err := c.scopedRequest("PATCH", "/v1/disk/resources?"+values.Encode(), body) + if err != nil { + return err + } + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return err + } + if err := CheckAPIError(resp); err != nil { + return err + } + + //If needed we can read response and check if custom_property is set. + + return nil +} diff --git a/yandex/api/delete.go b/yandex/api/delete.go new file mode 100644 index 000000000..0159913fc --- /dev/null +++ b/yandex/api/delete.go @@ -0,0 +1,26 @@ +package src + +import ( + "net/url" + "strconv" +) + +// Delete will remove specified file/folder from Yandex Disk +func (c *Client) Delete(remotePath string, permanently bool) error { + + values := url.Values{} + values.Add("permanently", strconv.FormatBool(permanently)) + values.Add("path", remotePath) + urlPath := "/v1/disk/resources?" + values.Encode() + fullURL := RootAddr + if urlPath[:1] != "/" { + fullURL += "/" + urlPath + } else { + fullURL += urlPath + } + + if err := c.PerformDelete(fullURL); err != nil { + return err + } + return nil +} diff --git a/yandex/api/disk_info_request.go b/yandex/api/disk_info_request.go new file mode 100644 index 000000000..c61518352 --- /dev/null +++ b/yandex/api/disk_info_request.go @@ -0,0 +1,48 @@ +package src + +import "encoding/json" + +//DiskInfoRequest type +type DiskInfoRequest struct { + client *Client + HTTPRequest *HTTPRequest +} + +func (req *DiskInfoRequest) request() *HTTPRequest { + return req.HTTPRequest +} + +//DiskInfoResponse struct is returned by the API for DiskInfo request. +type DiskInfoResponse struct { + TrashSize uint64 `json:"TrashSize"` + TotalSpace uint64 `json:"TotalSpace"` + UsedSpace uint64 `json:"UsedSpace"` + SystemFolders map[string]string `json:"SystemFolders"` +} + +//NewDiskInfoRequest create new DiskInfo Request +func (c *Client) NewDiskInfoRequest() *DiskInfoRequest { + return &DiskInfoRequest{ + client: c, + HTTPRequest: createGetRequest(c, "/", nil), + } +} + +//Exec run DiskInfo Request +func (req *DiskInfoRequest) Exec() (*DiskInfoResponse, error) { + data, err := req.request().run(req.client) + if err != nil { + return nil, err + } + + var info DiskInfoResponse + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + if info.SystemFolders == nil { + info.SystemFolders = make(map[string]string) + } + + return &info, nil +} diff --git a/yandex/api/download.go b/yandex/api/download.go new file mode 100644 index 000000000..6eed0c15f --- /dev/null +++ b/yandex/api/download.go @@ -0,0 +1,66 @@ +package src + +import ( + "encoding/json" + "io" + "net/url" +) + +// DownloadResponse struct is returned by the API for Download request. +type DownloadResponse struct { + HRef string `json:"href"` + Method string `json:"method"` + Templated bool `json:"templated"` +} + +// Download will get specified data from Yandex.Disk. +func (c *Client) Download(remotePath string) (io.ReadCloser, error) { //io.Writer + ur, err := c.DownloadRequest(remotePath) + if err != nil { + return nil, err + } + return c.PerformDownload(ur.HRef) +} + +// DownloadRequest will make an download request and return a URL to download data to. +func (c *Client) DownloadRequest(remotePath string) (*DownloadResponse, error) { + values := url.Values{} + values.Add("path", remotePath) + + req, err := c.scopedRequest("GET", "/v1/disk/resources/download?"+values.Encode(), nil) + if err != nil { + return nil, err + } + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + if err := CheckAPIError(resp); err != nil { + return nil, err + } + defer CheckClose(resp.Body, &err) + + ur, err := ParseDownloadResponse(resp.Body) + if err != nil { + return nil, err + } + + return ur, nil +} + +// ParseDownloadResponse tries to read and parse DownloadResponse struct. +func ParseDownloadResponse(data io.Reader) (*DownloadResponse, error) { + dec := json.NewDecoder(data) + var ur DownloadResponse + + if err := dec.Decode(&ur); err == io.EOF { + // ok + } else if err != nil { + return nil, err + } + + // TODO: check if there is any trash data after JSON and crash if there is. + + return &ur, nil +} diff --git a/yandex/api/error.go b/yandex/api/error.go new file mode 100644 index 000000000..d29137155 --- /dev/null +++ b/yandex/api/error.go @@ -0,0 +1,84 @@ +package src + +//from yadisk + +import ( + "encoding/json" + "fmt" + "io" + "net/http" +) + +// ErrorResponse represents erroneous API response. +// Implements go's built in `error`. +type ErrorResponse struct { + ErrorName string `json:"error"` + Description string `json:"description"` + Message string `json:"message"` + + StatusCode int `json:""` +} + +func (e *ErrorResponse) Error() string { + return fmt.Sprintf("[%d - %s] %s (%s)", e.StatusCode, e.ErrorName, e.Description, e.Message) +} + +// ProccessErrorResponse tries to represent data passed as +// an ErrorResponse object. +func ProccessErrorResponse(data io.Reader) (*ErrorResponse, error) { + dec := json.NewDecoder(data) + var errorResponse ErrorResponse + + if err := dec.Decode(&errorResponse); err == io.EOF { + // ok + } else if err != nil { + return nil, err + } + + // TODO: check if there is any trash data after JSON and crash if there is. + + return &errorResponse, nil +} + +// CheckAPIError is a convenient function to turn erroneous +// API response into go error. +func CheckAPIError(resp *http.Response) error { + if resp.StatusCode >= 200 && resp.StatusCode < 400 { + return nil + } + + errorResponse, err := ProccessErrorResponse(resp.Body) + if err != nil { + return err + } + errorResponse.StatusCode = resp.StatusCode + + defer CheckClose(resp.Body, &err) + + return errorResponse +} + +// ProccessErrorString tries to represent data passed as +// an ErrorResponse object. +func ProccessErrorString(data string) (*ErrorResponse, error) { + var errorResponse ErrorResponse + if err := json.Unmarshal([]byte(data), &errorResponse); err == nil { + // ok + } else if err != nil { + return nil, err + } + + // TODO: check if there is any trash data after JSON and crash if there is. + + return &errorResponse, nil +} + +// ParseAPIError Parse json error response from API +func (c *Client) ParseAPIError(jsonErr string) (string, error) { //ErrorName + errorResponse, err := ProccessErrorString(jsonErr) + if err != nil { + return err.Error(), err + } + + return errorResponse.ErrorName, nil +} diff --git a/yandex/api/errors.go b/yandex/api/errors.go new file mode 100644 index 000000000..1f0278824 --- /dev/null +++ b/yandex/api/errors.go @@ -0,0 +1,14 @@ +package src + +import "encoding/json" + +//DiskClientError struct +type DiskClientError struct { + Description string `json:"Description"` + Code string `json:"Error"` +} + +func (e DiskClientError) Error() string { + b, _ := json.Marshal(e) + return string(b) +} diff --git a/yandex/api/files_resource_list.go b/yandex/api/files_resource_list.go new file mode 100644 index 000000000..0af54294d --- /dev/null +++ b/yandex/api/files_resource_list.go @@ -0,0 +1,8 @@ +package src + +// FilesResourceListResponse struct is returned by the API for requests. +type FilesResourceListResponse struct { + Items []ResourceInfoResponse `json:"items"` + Limit *uint64 `json:"limit"` + Offset *uint64 `json:"offset"` +} diff --git a/yandex/api/flat_file_list_request.go b/yandex/api/flat_file_list_request.go new file mode 100644 index 000000000..ce047fa3e --- /dev/null +++ b/yandex/api/flat_file_list_request.go @@ -0,0 +1,78 @@ +package src + +import ( + "encoding/json" + "strings" +) + +// FlatFileListRequest struct client for FlatFileList Request +type FlatFileListRequest struct { + client *Client + HTTPRequest *HTTPRequest +} + +// FlatFileListRequestOptions struct - options for request +type FlatFileListRequestOptions struct { + MediaType []MediaType + Limit *uint32 + Offset *uint32 + Fields []string + PreviewSize *PreviewSize + PreviewCrop *bool +} + +// Request get request +func (req *FlatFileListRequest) Request() *HTTPRequest { + return req.HTTPRequest +} + +// NewFlatFileListRequest create new FlatFileList Request +func (c *Client) NewFlatFileListRequest(options ...FlatFileListRequestOptions) *FlatFileListRequest { + var parameters = make(map[string]interface{}) + if len(options) > 0 { + opt := options[0] + if opt.Limit != nil { + parameters["limit"] = *opt.Limit + } + if opt.Offset != nil { + parameters["offset"] = *opt.Offset + } + if opt.Fields != nil { + parameters["fields"] = strings.Join(opt.Fields, ",") + } + if opt.PreviewSize != nil { + parameters["preview_size"] = opt.PreviewSize.String() + } + if opt.PreviewCrop != nil { + parameters["preview_crop"] = *opt.PreviewCrop + } + if opt.MediaType != nil { + var strMediaTypes = make([]string, len(opt.MediaType)) + for i, t := range opt.MediaType { + strMediaTypes[i] = t.String() + } + parameters["media_type"] = strings.Join(strMediaTypes, ",") + } + } + return &FlatFileListRequest{ + client: c, + HTTPRequest: createGetRequest(c, "/resources/files", parameters), + } +} + +// Exec run FlatFileList Request +func (req *FlatFileListRequest) Exec() (*FilesResourceListResponse, error) { + data, err := req.Request().run(req.client) + if err != nil { + return nil, err + } + var info FilesResourceListResponse + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + if cap(info.Items) == 0 { + info.Items = []ResourceInfoResponse{} + } + return &info, nil +} diff --git a/yandex/api/http_request.go b/yandex/api/http_request.go new file mode 100644 index 000000000..640b37981 --- /dev/null +++ b/yandex/api/http_request.go @@ -0,0 +1,28 @@ +package src + +// HTTPRequest struct +type HTTPRequest struct { + Method string + Path string + Parameters map[string]interface{} + Headers map[string][]string +} + +func createGetRequest(client *Client, path string, params map[string]interface{}) *HTTPRequest { + return createRequest(client, "GET", path, params) +} + +func createPostRequest(client *Client, path string, params map[string]interface{}) *HTTPRequest { + return createRequest(client, "POST", path, params) +} + +func createRequest(client *Client, method string, path string, parameters map[string]interface{}) *HTTPRequest { + var headers = make(map[string][]string) + headers["Authorization"] = []string{"OAuth " + client.token} + return &HTTPRequest{ + Method: method, + Path: path, + Parameters: parameters, + Headers: headers, + } +} diff --git a/yandex/api/last_uploaded_resource_list.go b/yandex/api/last_uploaded_resource_list.go new file mode 100644 index 000000000..8c667e6bf --- /dev/null +++ b/yandex/api/last_uploaded_resource_list.go @@ -0,0 +1,7 @@ +package src + +// LastUploadedResourceListResponse struct +type LastUploadedResourceListResponse struct { + Items []ResourceInfoResponse `json:"items"` + Limit *uint64 `json:"limit"` +} diff --git a/yandex/api/last_uploaded_resource_list_request.go b/yandex/api/last_uploaded_resource_list_request.go new file mode 100644 index 000000000..f2c3fee95 --- /dev/null +++ b/yandex/api/last_uploaded_resource_list_request.go @@ -0,0 +1,74 @@ +package src + +import ( + "encoding/json" + "strings" +) + +// LastUploadedResourceListRequest struct +type LastUploadedResourceListRequest struct { + client *Client + HTTPRequest *HTTPRequest +} + +// LastUploadedResourceListRequestOptions struct +type LastUploadedResourceListRequestOptions struct { + MediaType []MediaType + Limit *uint32 + Fields []string + PreviewSize *PreviewSize + PreviewCrop *bool +} + +// Request return request +func (req *LastUploadedResourceListRequest) Request() *HTTPRequest { + return req.HTTPRequest +} + +// NewLastUploadedResourceListRequest create new LastUploadedResourceList Request +func (c *Client) NewLastUploadedResourceListRequest(options ...LastUploadedResourceListRequestOptions) *LastUploadedResourceListRequest { + var parameters = make(map[string]interface{}) + if len(options) > 0 { + opt := options[0] + if opt.Limit != nil { + parameters["limit"] = opt.Limit + } + if opt.Fields != nil { + parameters["fields"] = strings.Join(opt.Fields, ",") + } + if opt.PreviewSize != nil { + parameters["preview_size"] = opt.PreviewSize.String() + } + if opt.PreviewCrop != nil { + parameters["preview_crop"] = opt.PreviewCrop + } + if opt.MediaType != nil { + var strMediaTypes = make([]string, len(opt.MediaType)) + for i, t := range opt.MediaType { + strMediaTypes[i] = t.String() + } + parameters["media_type"] = strings.Join(strMediaTypes, ",") + } + } + return &LastUploadedResourceListRequest{ + client: c, + HTTPRequest: createGetRequest(c, "/resources/last-uploaded", parameters), + } +} + +// Exec run LastUploadedResourceList Request +func (req *LastUploadedResourceListRequest) Exec() (*LastUploadedResourceListResponse, error) { + data, err := req.Request().run(req.client) + if err != nil { + return nil, err + } + var info LastUploadedResourceListResponse + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + if cap(info.Items) == 0 { + info.Items = []ResourceInfoResponse{} + } + return &info, nil +} diff --git a/yandex/api/media_type.go b/yandex/api/media_type.go new file mode 100644 index 000000000..5f913e001 --- /dev/null +++ b/yandex/api/media_type.go @@ -0,0 +1,144 @@ +package src + +// MediaType struct - media types +type MediaType struct { + mediaType string +} + +// Audio - media type +func (m *MediaType) Audio() *MediaType { + return &MediaType{ + mediaType: "audio", + } +} + +// Backup - media type +func (m *MediaType) Backup() *MediaType { + return &MediaType{ + mediaType: "backup", + } +} + +// Book - media type +func (m *MediaType) Book() *MediaType { + return &MediaType{ + mediaType: "book", + } +} + +// Compressed - media type +func (m *MediaType) Compressed() *MediaType { + return &MediaType{ + mediaType: "compressed", + } +} + +// Data - media type +func (m *MediaType) Data() *MediaType { + return &MediaType{ + mediaType: "data", + } +} + +// Development - media type +func (m *MediaType) Development() *MediaType { + return &MediaType{ + mediaType: "development", + } +} + +// Diskimage - media type +func (m *MediaType) Diskimage() *MediaType { + return &MediaType{ + mediaType: "diskimage", + } +} + +// Document - media type +func (m *MediaType) Document() *MediaType { + return &MediaType{ + mediaType: "document", + } +} + +// Encoded - media type +func (m *MediaType) Encoded() *MediaType { + return &MediaType{ + mediaType: "encoded", + } +} + +// Executable - media type +func (m *MediaType) Executable() *MediaType { + return &MediaType{ + mediaType: "executable", + } +} + +// Flash - media type +func (m *MediaType) Flash() *MediaType { + return &MediaType{ + mediaType: "flash", + } +} + +// Font - media type +func (m *MediaType) Font() *MediaType { + return &MediaType{ + mediaType: "font", + } +} + +// Image - media type +func (m *MediaType) Image() *MediaType { + return &MediaType{ + mediaType: "image", + } +} + +// Settings - media type +func (m *MediaType) Settings() *MediaType { + return &MediaType{ + mediaType: "settings", + } +} + +// Spreadsheet - media type +func (m *MediaType) Spreadsheet() *MediaType { + return &MediaType{ + mediaType: "spreadsheet", + } +} + +// Text - media type +func (m *MediaType) Text() *MediaType { + return &MediaType{ + mediaType: "text", + } +} + +// Unknown - media type +func (m *MediaType) Unknown() *MediaType { + return &MediaType{ + mediaType: "unknown", + } +} + +// Video - media type +func (m *MediaType) Video() *MediaType { + return &MediaType{ + mediaType: "video", + } +} + +// Web - media type +func (m *MediaType) Web() *MediaType { + return &MediaType{ + mediaType: "web", + } +} + +// String - media type +func (m *MediaType) String() string { + return m.mediaType +} diff --git a/yandex/api/mkdir.go b/yandex/api/mkdir.go new file mode 100644 index 000000000..e9703f4d5 --- /dev/null +++ b/yandex/api/mkdir.go @@ -0,0 +1,21 @@ +package src + +import ( + "net/url" +) + +// Mkdir will make specified folder on Yandex Disk +func (c *Client) Mkdir(remotePath string) (int, string, error) { + + values := url.Values{} + values.Add("path", remotePath) // only one current folder will be created. Not all the folders in the path. + urlPath := "/v1/disk/resources?" + values.Encode() + fullURL := RootAddr + if urlPath[:1] != "/" { + fullURL += "/" + urlPath + } else { + fullURL += urlPath + } + + return c.PerformMkdir(fullURL) +} diff --git a/yandex/api/performdelete.go b/yandex/api/performdelete.go new file mode 100644 index 000000000..5aa0b616b --- /dev/null +++ b/yandex/api/performdelete.go @@ -0,0 +1,34 @@ +package src + +import ( + "fmt" + "io/ioutil" + "net/http" +) + +// PerformDelete does the actual delete via DELETE request. +func (c *Client) PerformDelete(url string) error { + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return err + } + + //set access token and headers + c.setRequestScope(req) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + //204 - resource deleted. + //202 - folder not empty, content will be deleted soon (async delete). + if resp.StatusCode != 204 && resp.StatusCode != 202 { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + return fmt.Errorf("delete error [%d]: %s", resp.StatusCode, string(body[:])) + } + return nil +} diff --git a/yandex/api/performdownload.go b/yandex/api/performdownload.go new file mode 100644 index 000000000..590f7f042 --- /dev/null +++ b/yandex/api/performdownload.go @@ -0,0 +1,33 @@ +package src + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" +) + +// PerformDownload does the actual download via unscoped PUT request. +func (c *Client) PerformDownload(url string) (io.ReadCloser, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + //c.setRequestScope(req) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + if resp.StatusCode != 200 { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + defer CheckClose(resp.Body, &err) + return nil, fmt.Errorf("download error [%d]: %s", resp.StatusCode, string(body[:])) + } + return resp.Body, err +} diff --git a/yandex/api/performmkdir.go b/yandex/api/performmkdir.go new file mode 100644 index 000000000..3a6f85b25 --- /dev/null +++ b/yandex/api/performmkdir.go @@ -0,0 +1,33 @@ +package src + +import ( + "fmt" + "io/ioutil" + "net/http" +) + +// PerformMkdir does the actual mkdir via PUT request. +func (c *Client) PerformMkdir(url string) (int, string, error) { + req, err := http.NewRequest("PUT", url, nil) + if err != nil { + return 0, "", err + } + + //set access token and headers + c.setRequestScope(req) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0, "", err + } + + if resp.StatusCode != 201 { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return 0, "", err + } + //third parameter is the json error response body + return resp.StatusCode, string(body[:]), fmt.Errorf("Create Folder error [%d]: %s", resp.StatusCode, string(body[:])) + } + return resp.StatusCode, "", nil +} diff --git a/yandex/api/performupload.go b/yandex/api/performupload.go new file mode 100644 index 000000000..d6bda4aed --- /dev/null +++ b/yandex/api/performupload.go @@ -0,0 +1,36 @@ +package src + +//from yadisk + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" +) + +// PerformUpload does the actual upload via unscoped PUT request. +func (c *Client) PerformUpload(url string, data io.Reader) error { + req, err := http.NewRequest("PUT", url, data) + if err != nil { + return err + } + + //c.setRequestScope(req) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer CheckClose(resp.Body, &err) + + if resp.StatusCode != 201 { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return fmt.Errorf("upload error [%d]: %s", resp.StatusCode, string(body[:])) + } + return nil +} diff --git a/yandex/api/preview_size.go b/yandex/api/preview_size.go new file mode 100644 index 000000000..3fba5c889 --- /dev/null +++ b/yandex/api/preview_size.go @@ -0,0 +1,75 @@ +package src + +import "fmt" + +// PreviewSize struct +type PreviewSize struct { + size string +} + +// PredefinedSizeS - set preview size +func (s *PreviewSize) PredefinedSizeS() *PreviewSize { + return &PreviewSize{ + size: "S", + } +} + +// PredefinedSizeM - set preview size +func (s *PreviewSize) PredefinedSizeM() *PreviewSize { + return &PreviewSize{ + size: "M", + } +} + +// PredefinedSizeL - set preview size +func (s *PreviewSize) PredefinedSizeL() *PreviewSize { + return &PreviewSize{ + size: "L", + } +} + +// PredefinedSizeXL - set preview size +func (s *PreviewSize) PredefinedSizeXL() *PreviewSize { + return &PreviewSize{ + size: "XL", + } +} + +// PredefinedSizeXXL - set preview size +func (s *PreviewSize) PredefinedSizeXXL() *PreviewSize { + return &PreviewSize{ + size: "XXL", + } +} + +// PredefinedSizeXXXL - set preview size +func (s *PreviewSize) PredefinedSizeXXXL() *PreviewSize { + return &PreviewSize{ + size: "XXXL", + } +} + +// ExactWidth - set preview size +func (s *PreviewSize) ExactWidth(width uint32) *PreviewSize { + return &PreviewSize{ + size: fmt.Sprintf("%dx", width), + } +} + +// ExactHeight - set preview size +func (s *PreviewSize) ExactHeight(height uint32) *PreviewSize { + return &PreviewSize{ + size: fmt.Sprintf("x%d", height), + } +} + +// ExactSize - set preview size +func (s *PreviewSize) ExactSize(width uint32, height uint32) *PreviewSize { + return &PreviewSize{ + size: fmt.Sprintf("%dx%d", width, height), + } +} + +func (s *PreviewSize) String() string { + return s.size +} diff --git a/yandex/api/resource.go b/yandex/api/resource.go new file mode 100644 index 000000000..6d9adb5b7 --- /dev/null +++ b/yandex/api/resource.go @@ -0,0 +1,19 @@ +package src + +//ResourceInfoResponse struct is returned by the API for metedata requests. +type ResourceInfoResponse struct { + PublicKey string `json:"public_key"` + Name string `json:"name"` + Created string `json:"created"` + CustomProperties map[string]interface{} `json:"custom_properties"` + Preview string `json:"preview"` + PublicURL string `json:"public_url"` + OriginPath string `json:"origin_path"` + Modified string `json:"modified"` + Path string `json:"path"` + Md5 string `json:"md5"` + ResourceType string `json:"type"` + MimeType string `json:"mime_type"` + Size uint64 `json:"size"` + Embedded *ResourceListResponse `json:"_embedded"` +} diff --git a/yandex/api/resource_info_request.go b/yandex/api/resource_info_request.go new file mode 100644 index 000000000..5ad601794 --- /dev/null +++ b/yandex/api/resource_info_request.go @@ -0,0 +1,45 @@ +package src + +import "encoding/json" + +// ResourceInfoRequest struct +type ResourceInfoRequest struct { + client *Client + HTTPRequest *HTTPRequest +} + +// Request of ResourceInfoRequest +func (req *ResourceInfoRequest) Request() *HTTPRequest { + return req.HTTPRequest +} + +// NewResourceInfoRequest create new ResourceInfo Request +func (c *Client) NewResourceInfoRequest(path string, options ...ResourceInfoRequestOptions) *ResourceInfoRequest { + return &ResourceInfoRequest{ + client: c, + HTTPRequest: createResourceInfoRequest(c, "/resources", path, options...), + } +} + +// Exec run ResourceInfo Request +func (req *ResourceInfoRequest) Exec() (*ResourceInfoResponse, error) { + data, err := req.Request().run(req.client) + if err != nil { + return nil, err + } + + var info ResourceInfoResponse + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + if info.CustomProperties == nil { + info.CustomProperties = make(map[string]interface{}) + } + if info.Embedded != nil { + if cap(info.Embedded.Items) == 0 { + info.Embedded.Items = []ResourceInfoResponse{} + } + } + return &info, nil +} diff --git a/yandex/api/resource_info_request_helpers.go b/yandex/api/resource_info_request_helpers.go new file mode 100644 index 000000000..941fe066a --- /dev/null +++ b/yandex/api/resource_info_request_helpers.go @@ -0,0 +1,33 @@ +package src + +import "strings" + +func createResourceInfoRequest(c *Client, + apiPath string, + path string, + options ...ResourceInfoRequestOptions) *HTTPRequest { + var parameters = make(map[string]interface{}) + parameters["path"] = path + if len(options) > 0 { + opt := options[0] + if opt.SortMode != nil { + parameters["sort"] = opt.SortMode.String() + } + if opt.Limit != nil { + parameters["limit"] = opt.Limit + } + if opt.Offset != nil { + parameters["offset"] = opt.Offset + } + if opt.Fields != nil { + parameters["fields"] = strings.Join(opt.Fields, ",") + } + if opt.PreviewSize != nil { + parameters["preview_size"] = opt.PreviewSize.String() + } + if opt.PreviewCrop != nil { + parameters["preview_crop"] = opt.PreviewCrop + } + } + return createGetRequest(c, apiPath, parameters) +} diff --git a/yandex/api/resource_info_request_options.go b/yandex/api/resource_info_request_options.go new file mode 100644 index 000000000..ffe07d44e --- /dev/null +++ b/yandex/api/resource_info_request_options.go @@ -0,0 +1,11 @@ +package src + +// ResourceInfoRequestOptions struct +type ResourceInfoRequestOptions struct { + SortMode *SortMode + Limit *uint32 + Offset *uint32 + Fields []string + PreviewSize *PreviewSize + PreviewCrop *bool +} diff --git a/yandex/api/resource_list.go b/yandex/api/resource_list.go new file mode 100644 index 000000000..f15caf1c4 --- /dev/null +++ b/yandex/api/resource_list.go @@ -0,0 +1,12 @@ +package src + +// ResourceListResponse struct +type ResourceListResponse struct { + Sort *SortMode `json:"sort"` + PublicKey string `json:"public_key"` + Items []ResourceInfoResponse `json:"items"` + Path string `json:"path"` + Limit *uint64 `json:"limit"` + Offset *uint64 `json:"offset"` + Total *uint64 `json:"total"` +} diff --git a/yandex/api/sort_mode.go b/yandex/api/sort_mode.go new file mode 100644 index 000000000..41e74e66f --- /dev/null +++ b/yandex/api/sort_mode.go @@ -0,0 +1,79 @@ +package src + +import "strings" + +// SortMode struct - sort mode +type SortMode struct { + mode string +} + +// Default - sort mode +func (m *SortMode) Default() *SortMode { + return &SortMode{ + mode: "", + } +} + +// ByName - sort mode +func (m *SortMode) ByName() *SortMode { + return &SortMode{ + mode: "name", + } +} + +// ByPath - sort mode +func (m *SortMode) ByPath() *SortMode { + return &SortMode{ + mode: "path", + } +} + +// ByCreated - sort mode +func (m *SortMode) ByCreated() *SortMode { + return &SortMode{ + mode: "created", + } +} + +// ByModified - sort mode +func (m *SortMode) ByModified() *SortMode { + return &SortMode{ + mode: "modified", + } +} + +// BySize - sort mode +func (m *SortMode) BySize() *SortMode { + return &SortMode{ + mode: "size", + } +} + +// Reverse - sort mode +func (m *SortMode) Reverse() *SortMode { + if strings.HasPrefix(m.mode, "-") { + return &SortMode{ + mode: m.mode[1:], + } + } + return &SortMode{ + mode: "-" + m.mode, + } +} + +func (m *SortMode) String() string { + return m.mode +} + +// UnmarshalJSON sort mode +func (m *SortMode) UnmarshalJSON(value []byte) error { + if value == nil || len(value) == 0 { + m.mode = "" + return nil + } + m.mode = string(value) + if strings.HasPrefix(m.mode, "\"") && strings.HasSuffix(m.mode, "\"") { + m.mode = m.mode[1 : len(m.mode)-1] + } + return nil +} diff --git a/yandex/api/trash_resource_info_request.go b/yandex/api/trash_resource_info_request.go new file mode 100644 index 000000000..3911223b8 --- /dev/null +++ b/yandex/api/trash_resource_info_request.go @@ -0,0 +1,45 @@ +package src + +import "encoding/json" + +// TrashResourceInfoRequest struct +type TrashResourceInfoRequest struct { + client *Client + HTTPRequest *HTTPRequest +} + +// Request of TrashResourceInfoRequest struct +func (req *TrashResourceInfoRequest) Request() *HTTPRequest { + return req.HTTPRequest +} + +// NewTrashResourceInfoRequest create new TrashResourceInfo Request +func (c *Client) NewTrashResourceInfoRequest(path string, options ...ResourceInfoRequestOptions) *TrashResourceInfoRequest { + return &TrashResourceInfoRequest{ + client: c, + HTTPRequest: createResourceInfoRequest(c, "/trash/resources", path, options...), + } +} + +// Exec run TrashResourceInfo Request +func (req *TrashResourceInfoRequest) Exec() (*ResourceInfoResponse, error) { + data, err := req.Request().run(req.client) + if err != nil { + return nil, err + } + + var info ResourceInfoResponse + err = json.Unmarshal(data, &info) + if err != nil { + return nil, err + } + if info.CustomProperties == nil { + info.CustomProperties = make(map[string]interface{}) + } + if info.Embedded != nil { + if cap(info.Embedded.Items) == 0 { + info.Embedded.Items = []ResourceInfoResponse{} + } + } + return &info, nil +} diff --git a/yandex/api/upload.go b/yandex/api/upload.go new file mode 100644 index 000000000..a31ec3341 --- /dev/null +++ b/yandex/api/upload.go @@ -0,0 +1,75 @@ +package src + +//from yadisk + +import ( + "encoding/json" + "io" + "net/url" + "strconv" +) + +// UploadResponse struct is returned by the API for upload request. +type UploadResponse struct { + HRef string `json:"href"` + Method string `json:"method"` + Templated bool `json:"templated"` +} + +// Upload will put specified data to Yandex.Disk. +func (c *Client) Upload(data io.Reader, remotePath string, overwrite bool) error { + ur, err := c.UploadRequest(remotePath, overwrite) + if err != nil { + return err + } + + if err := c.PerformUpload(ur.HRef, data); err != nil { + return err + } + + return nil +} + +// UploadRequest will make an upload request and return a URL to upload data to. +func (c *Client) UploadRequest(remotePath string, overwrite bool) (*UploadResponse, error) { + values := url.Values{} + values.Add("path", remotePath) + values.Add("overwrite", strconv.FormatBool(overwrite)) + + req, err := c.scopedRequest("GET", "/v1/disk/resources/upload?"+values.Encode(), nil) + if err != nil { + return nil, err + } + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, err + } + if err := CheckAPIError(resp); err != nil { + return nil, err + } + defer CheckClose(resp.Body, &err) + + ur, err := ParseUploadResponse(resp.Body) + if err != nil { + return nil, err + } + + return ur, nil +} + +// ParseUploadResponse tries to read and parse UploadResponse struct. +func ParseUploadResponse(data io.Reader) (*UploadResponse, error) { + dec := json.NewDecoder(data) + var ur UploadResponse + + if err := dec.Decode(&ur); err == io.EOF { + // ok + } else if err != nil { + return nil, err + } + + // TODO: check if there is any trash data after JSON and crash if there is. + + return &ur, nil +} diff --git a/yandex/yandex.go b/yandex/yandex.go new file mode 100644 index 000000000..4b279ee29 --- /dev/null +++ b/yandex/yandex.go @@ -0,0 +1,541 @@ +// Package yandex provides an interface to the Yandex Disk storage. +// +// dibu28 github.com/dibu28 +package yandex + +import ( + "encoding/json" + "fmt" + "io" + "log" + "path" + "path/filepath" + "strings" + "time" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/oauthutil" + yandex "github.com/ncw/rclone/yandex/api" + "golang.org/x/oauth2" +) + +//oAuth +const ( + rcloneClientID = "ac39b43b9eba4cae8ffb788c06d816a8" + rcloneClientSecret = "k8jKzZnMmM+Wx5jAksPAwYKPgImOiN+FhNKD09KBg9A=" +) + +// Globals +var ( + // Description of how to auth for this app + oauthConfig = &oauth2.Config{ + Endpoint: oauth2.Endpoint{ + AuthURL: "https://oauth.yandex.com/authorize", //same as https://oauth.yandex.ru/authorize + TokenURL: "https://oauth.yandex.com/token", //same as https://oauth.yandex.ru/token + }, + ClientID: rcloneClientID, + ClientSecret: fs.Reveal(rcloneClientSecret), + RedirectURL: oauthutil.RedirectURL, + } +) + +// Register with Fs +func init() { + fs.Register(&fs.Info{ + Name: "yandex", + NewFs: NewFs, + Config: func(name string) { + err := oauthutil.Config(name, oauthConfig) + if err != nil { + log.Fatalf("Failed to configure token: %v", err) + } + }, + Options: []fs.Option{{ + Name: oauthutil.ConfigClientID, + Help: "Yandex Client Id - leave blank normally.", + }, { + Name: oauthutil.ConfigClientSecret, + Help: "Yandex Client Secret - leave blank normally.", + }}, + }) +} + +// Fs represents a remote yandex +type Fs struct { + name string + yd *yandex.Client // client for rest api + root string //root path + diskRoot string //root path with "disk:/" container name + mkdircache map[string]int +} + +// Object describes a swift object +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + md5sum string // The MD5Sum of the object + bytes uint64 // Bytes in the object + modTime time.Time // Modified time of the object +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("Yandex %s", f.root) +} + +// read access token from ConfigFile string +func getAccessToken(name string) (*oauth2.Token, error) { + // Read the token from the config file + tokenConfig := fs.ConfigFile.MustValue(name, "token") + //Get access token from config string + decoder := json.NewDecoder(strings.NewReader(tokenConfig)) + var result *oauth2.Token + err := decoder.Decode(&result) + if err != nil { + return nil, err + } + return result, nil +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(name, root string) (fs.Fs, error) { + //read access token from config + token, err := getAccessToken(name) + if err != nil { + return nil, err + } + + //create new client + yandexDisk := yandex.NewClient(token.AccessToken) + + f := &Fs{ + yd: yandexDisk, + } + + f.setRoot(root) + + //limited fs + // Check to see if the object exists and is a file + //request object meta info + var opt2 yandex.ResourceInfoRequestOptions + if ResourceInfoResponse, err := yandexDisk.NewResourceInfoRequest(root, opt2).Exec(); err != nil { + //return err + } else { + if ResourceInfoResponse.ResourceType == "file" { + //limited fs + remote := path.Base(root) + f.setRoot(path.Dir(root)) + + obj := f.newFsObjectWithInfo(remote, ResourceInfoResponse) + // return a Fs Limited to this object + return fs.NewLimited(f, obj), nil + } + } + + return f, nil +} + +// Sets root in f +func (f *Fs) setRoot(root string) { + //Set root path + f.root = strings.Trim(root, "/") + //Set disk root path. + //Adding "disk:" to root path as all paths on disk start with it + var diskRoot = "" + if f.root == "" { + diskRoot = "disk:/" + } else { + diskRoot = "disk:/" + f.root + "/" + } + f.diskRoot = diskRoot +} + +// list the objects into the function supplied +// +// If directories is set it only sends directories +func (f *Fs) list(directories bool, fn func(string, yandex.ResourceInfoResponse)) { + //request files list. list is divided into pages. We send request for each page + //items per page is limited by limit + //TODO may be add config parameter for the items per page limit + var limit uint32 = 1000 // max number of object per request + var itemsCount uint32 //number of items per page in response + var offset uint32 //for the next page of request + // yandex disk api request options + var opt yandex.FlatFileListRequestOptions + opt.Limit = &limit + opt.Offset = &offset + //query each page of list until itemCount is less then limit + for { + //send request + info, err := f.yd.NewFlatFileListRequest(opt).Exec() + if err != nil { + fs.Stats.Error() + fs.ErrorLog(f, "Couldn't list: %s", err) + return + } + itemsCount = uint32(len(info.Items)) + + //list files + for _, item := range info.Items { + // filter file list and get only files we need + if strings.HasPrefix(item.Path, f.diskRoot) { + //trim root folder from filename + var name = strings.TrimPrefix(item.Path, f.diskRoot) + fn(name, item) + } + } + + //offset for the next page of items + offset += itemsCount + //check if we reached end of list + if itemsCount < limit { + break + } + } +} + +// List walks the path returning a channel of FsObjects +func (f *Fs) List() fs.ObjectsChan { + out := make(fs.ObjectsChan, fs.Config.Checkers) + // List the objects + go func() { + defer close(out) + f.list(false, func(remote string, object yandex.ResourceInfoResponse) { + if fs := f.newFsObjectWithInfo(remote, &object); fs != nil { + out <- fs + } + }) + }() + return out +} + +// NewFsObject returns an Object from a path +// +// May return nil if an error occurred +func (f *Fs) NewFsObject(remote string) fs.Object { + return f.newFsObjectWithInfo(remote, nil) +} + +// Return an FsObject from a path +// +// May return nil if an error occurred +func (f *Fs) newFsObjectWithInfo(remote string, info *yandex.ResourceInfoResponse) fs.Object { + o := &Object{ + fs: f, + remote: remote, + } + if info != nil { + o.setMetaData(info) + } else { + err := o.readMetaData() + if err != nil { + fs.ErrorLog(f, "Couldn't get object '%s' metadata: %s", o.remotePath(), err) + return nil + } + } + return o +} + +// setMetaData sets the fs data from a storage.Object +func (o *Object) setMetaData(info *yandex.ResourceInfoResponse) { + o.bytes = info.Size + o.md5sum = info.Md5 + + if info.CustomProperties["rclone_modified"] == nil { + //read modTime from Modified property of object + t, err := time.Parse(time.RFC3339Nano, info.Modified) + if err != nil { + return + } + o.modTime = t + } else { + // interface{} to string type assertion + if modtimestr, ok := info.CustomProperties["rclone_modified"].(string); ok { + //read modTime from rclone_modified custom_property of object + t, err := time.Parse(time.RFC3339Nano, modtimestr) + if err != nil { + return + } + o.modTime = t + } else { + return //if it is not a string + } + } +} + +// readMetaData gets the info if it hasn't already been fetched +func (o *Object) readMetaData() (err error) { + //request meta info + var opt2 yandex.ResourceInfoRequestOptions + ResourceInfoResponse, err := o.fs.yd.NewResourceInfoRequest(o.remotePath(), opt2).Exec() + if err != nil { + return err + } + o.setMetaData(ResourceInfoResponse) + return nil +} + +// ListDir walks the path returning a channel of FsObjects +func (f *Fs) ListDir() fs.DirChan { + out := make(fs.DirChan, fs.Config.Checkers) + go func() { + defer close(out) + + //request object meta info + var opt yandex.ResourceInfoRequestOptions + ResourceInfoResponse, err := f.yd.NewResourceInfoRequest(f.diskRoot, opt).Exec() + if err != nil { + return + } + if ResourceInfoResponse.ResourceType == "dir" { + //list all subdirs + for _, element := range ResourceInfoResponse.Embedded.Items { + if element.ResourceType == "dir" { + t, err := time.Parse(time.RFC3339Nano, element.Modified) + if err != nil { + return + } + out <- &fs.Dir{ + Name: element.Name, + When: t, + Bytes: int64(element.Size), + Count: -1, + } + } + } + } + + }() + return out +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + bytes: uint64(size), + modTime: modTime, + } + //TODO maybe read metadata after upload to check if file uploaded successfully + return o, o.Update(in, modTime, size) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir() error { + return mkDirFullPath(f.yd, f.diskRoot) +} + +// Rmdir deletes the container +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir() error { + return f.purgeCheck(true) +} + +// purgeCheck remotes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(check bool) error { + if check { + //to comply with rclone logic we check if the directory is empty before delete. + //send request to get list of objects in this directory. + var opt yandex.ResourceInfoRequestOptions + ResourceInfoResponse, err := f.yd.NewResourceInfoRequest(f.diskRoot, opt).Exec() + if err != nil { + return fmt.Errorf("Rmdir failed: %s", err) + } + if len(ResourceInfoResponse.Embedded.Items) != 0 { + return fmt.Errorf("Rmdir failed: Directory not empty") + } + } + //delete directory + return f.yd.Delete(f.diskRoot, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// Purge deletes all the files and the container +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge() error { + return f.purgeCheck(false) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Fs { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Md5sum returns the Md5sum of an object returning a lowercase hex string +func (o *Object) Md5sum() (string, error) { + return o.md5sum, nil +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + var size = int64(o.bytes) //need to cast from uint64 in yandex disk to int64 in rclone. can cause overflow + return size +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime() time.Time { + err := o.readMetaData() + if err != nil { + fs.Log(o, "Failed to read metadata: %s", err) + return time.Now() + } + return o.modTime +} + +// Open an object for read +func (o *Object) Open() (in io.ReadCloser, err error) { + return o.fs.yd.Download(o.remotePath()) +} + +// Remove an object +func (o *Object) Remove() error { + return o.fs.yd.Delete(o.remotePath(), true) +} + +// SetModTime sets the modification time of the local fs object +// +// Commits the datastore +func (o *Object) SetModTime(modTime time.Time) { + remote := o.remotePath() + //set custom_property 'rclone_modified' of object to modTime + err := o.fs.yd.SetCustomProperty(remote, "rclone_modified", modTime.Format(time.RFC3339Nano)) + if err != nil { + return + } + return +} + +// Storable returns whether this object is storable +func (o *Object) Storable() bool { + return true +} + +// Returns the remote path for the object +func (o *Object) remotePath() string { + return o.fs.diskRoot + o.remote +} + +// Update the already existing object +// +// Copy the reader into the object updating modTime and size +// +// The new object may have been created if an error is returned +func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error { + remote := o.remotePath() + //create full path to file before upload. + err1 := mkDirFullPath(o.fs.yd, remote) + if err1 != nil { + return err1 + } + //upload file + overwrite := true //overwrite existing file + err := o.fs.yd.Upload(in, remote, overwrite) + if err == nil { + //if file uploaded sucessfuly then return metadata + o.bytes = uint64(size) + o.modTime = modTime + o.md5sum = "" // according to unit tests after put the md5 is empty. + //and set modTime of uploaded file + o.SetModTime(modTime) + } + return err +} + +// utility funcs------------------------------------------------------------------- + +// mkDirExecute execute mkdir +func mkDirExecute(client *yandex.Client, path string) (int, string, error) { + statusCode, jsonErrorString, err := client.Mkdir(path) + if statusCode == 409 { // dir already exist + return statusCode, jsonErrorString, err + } + if statusCode == 201 { // dir was created + return statusCode, jsonErrorString, err + } + if err != nil { + // error creating directory + log.Fatalf("Failed to create folder: %v", err) + return statusCode, jsonErrorString, err + } + return 0, "", nil +} + +//mkDirFullPath Creates Each Directory in the path if needed. Send request once for every directory in the path. +func mkDirFullPath(client *yandex.Client, path string) error { + //trim filename from path + dirString := strings.TrimSuffix(path, filepath.Base(path)) + //trim "disk:/" from path + dirString = strings.TrimPrefix(dirString, "disk:/") + + //1 Try to create directory first + if _, jsonErrorString, err := mkDirExecute(client, dirString); err != nil { + er2, _ := client.ParseAPIError(jsonErrorString) + if er2 != "DiskPathPointsToExistentDirectoryError" { + //2 if it fails then create all directories in the path from root. + dirs := strings.Split(dirString, "/") //path separator / + var mkdirpath = "/" //path separator / + for _, element := range dirs { + if element != "" { + mkdirpath += element + "/" //path separator / + _, _, err2 := mkDirExecute(client, mkdirpath) + if err2 != nil { + //we continue even if some directories exist. + } + } + } + } + } + return nil +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + //_ fs.Copier = (*Fs)(nil) + _ fs.Object = (*Object)(nil) +) diff --git a/yandex/yandex_test.go b/yandex/yandex_test.go new file mode 100644 index 000000000..c407ac746 --- /dev/null +++ b/yandex/yandex_test.go @@ -0,0 +1,56 @@ +// Test Yandex filesystem interface +// +// Automatically generated - DO NOT EDIT +// Regenerate with: make gen_tests +package yandex_test + +import ( + "testing" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fstest/fstests" + "github.com/ncw/rclone/yandex" +) + +func init() { + fstests.NilObject = fs.Object((*yandex.Object)(nil)) + fstests.RemoteName = "TestYandex:" +} + +// Generic tests for the Fs +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsNewFsObjectNotFound(t *testing.T) { fstests.TestFsNewFsObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRoot(t *testing.T) { fstests.TestFsListRoot(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewFsObject(t *testing.T) { fstests.TestFsNewFsObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectMd5sum(t *testing.T) { fstests.TestObjectMd5sum(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestLimitedFs(t *testing.T) { fstests.TestLimitedFs(t) } +func TestLimitedFsNotFound(t *testing.T) { fstests.TestLimitedFsNotFound(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }