From 8a774a3dd47cfe499e3ceaacaca3ac442e5a99c1 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 27 Jan 2019 13:33:21 +0000 Subject: [PATCH] webdav: support MD5 and SHA1 hashes with Owncloud and Nextcloud - fixes #2379 --- backend/webdav/api/types.go | 32 ++++++++++++--- backend/webdav/webdav.go | 79 ++++++++++++++++++++++++++++--------- docs/content/overview.md | 6 ++- docs/content/webdav.md | 6 ++- 4 files changed, 96 insertions(+), 27 deletions(-) diff --git a/backend/webdav/api/types.go b/backend/webdav/api/types.go index 4d248ac36..b0d2a57d5 100644 --- a/backend/webdav/api/types.go +++ b/backend/webdav/api/types.go @@ -10,6 +10,7 @@ import ( "time" "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/hash" ) const ( @@ -65,11 +66,12 @@ type Response struct { // Note that status collects all the status values for which we just // check the first is OK. type Prop struct { - Status []string `xml:"DAV: status"` - Name string `xml:"DAV: prop>displayname,omitempty"` - Type *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"` - Size int64 `xml:"DAV: prop>getcontentlength,omitempty"` - Modified Time `xml:"DAV: prop>getlastmodified,omitempty"` + Status []string `xml:"DAV: status"` + Name string `xml:"DAV: prop>displayname,omitempty"` + Type *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"` + Size int64 `xml:"DAV: prop>getcontentlength,omitempty"` + Modified Time `xml:"DAV: prop>getlastmodified,omitempty"` + Checksums []string `xml:"prop>checksums>checksum,omitempty"` } // Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200" @@ -95,6 +97,26 @@ func (p *Prop) StatusOK() bool { return false } +// Hashes returns a map of all checksums - may be nil +func (p *Prop) Hashes() (hashes map[hash.Type]string) { + if len(p.Checksums) == 0 { + return nil + } + hashes = make(map[hash.Type]string) + for _, checksums := range p.Checksums { + checksums = strings.ToLower(checksums) + for _, checksum := range strings.Split(checksums, " ") { + switch { + case strings.HasPrefix(checksum, "sha1:"): + hashes[hash.SHA1] = checksum[5:] + case strings.HasPrefix(checksum, "md5:"): + hashes[hash.MD5] = checksum[4:] + } + } + } + return hashes +} + // PropValue is a tagged name and value type PropValue struct { XMLName xml.Name `xml:""` diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go index 2738d6200..d2cd9facc 100644 --- a/backend/webdav/webdav.go +++ b/backend/webdav/webdav.go @@ -2,23 +2,13 @@ // object storage system. package webdav -// Owncloud: Getting Oc-Checksum: -// SHA1:f572d396fae9206628714fb2ce00f72e94f2258f on HEAD but not on -// nextcloud? - -// docs for file webdav -// https://docs.nextcloud.com/server/12/developer_manual/client_apis/WebDAV/index.html - -// indicates checksums can be set as metadata here -// https://github.com/nextcloud/server/issues/6129 -// owncloud seems to have checksums as metadata though - can read them - // SetModTime might be possible // https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file // ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option. // For example the ownCloud WebDAV server does it that way. import ( + "bytes" "encoding/xml" "fmt" "io" @@ -116,6 +106,7 @@ type Fs struct { canStream bool // set if can stream useOCMtime bool // set if can use X-OC-Mtime retryWithZeroDepth bool // some vendors (sharepoint) won't list files when Depth is 1 (our default) + hasChecksums bool // set if can use owncloud style checksums } // Object describes a webdav object @@ -127,7 +118,8 @@ type Object struct { hasMetaData bool // whether info below has been set size int64 // size of the object modTime time.Time // modification time of the object - sha1 string // SHA-1 of the object content + sha1 string // SHA-1 of the object content if known + md5 string // MD5 of the object content if known } // ------------------------------------------------------------ @@ -194,6 +186,9 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err }, NoRedirect: true, } + if f.hasChecksums { + opts.Body = bytes.NewBuffer(owncloudProps) + } var result api.Multistatus var resp *http.Response err = f.pacer.Call(func() (bool, error) { @@ -357,9 +352,11 @@ func (f *Fs) setQuirks(vendor string) error { f.canStream = true f.precision = time.Second f.useOCMtime = true + f.hasChecksums = true case "nextcloud": f.precision = time.Second f.useOCMtime = true + f.hasChecksums = true case "sharepoint": // To mount sharepoint, two Cookies are required // They have to be set instead of BasicAuth @@ -426,6 +423,22 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) { return f.newObjectWithInfo(remote, nil) } +// Read the normal props, plus the checksums +// +// SHA1:f572d396fae9206628714fb2ce00f72e94f2258f MD5:b1946ac92492d2347c6235b4d2611184 ADLER32:084b021f +var owncloudProps = []byte(` + + + + + + + + + + +`) + // list the objects into the function supplied // // If directories is set it only sends directories @@ -445,6 +458,9 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str "Depth": depth, }, } + if f.hasChecksums { + opts.Body = bytes.NewBuffer(owncloudProps) + } var result api.Multistatus var resp *http.Response err = f.pacer.Call(func() (bool, error) { @@ -847,6 +863,9 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { // Hashes returns the supported hash sets. func (f *Fs) Hashes() hash.Set { + if f.hasChecksums { + return hash.NewHashSet(hash.MD5, hash.SHA1) + } return hash.Set(hash.None) } @@ -870,12 +889,17 @@ func (o *Object) Remote() string { return o.remote } -// Hash returns the SHA-1 of an object returning a lowercase hex string +// Hash returns the SHA1 or MD5 of an object returning a lowercase hex string func (o *Object) Hash(t hash.Type) (string, error) { - if t != hash.SHA1 { - return "", hash.ErrUnsupported + if o.fs.hasChecksums { + switch t { + case hash.SHA1: + return o.sha1, nil + case hash.MD5: + return o.md5, nil + } } - return o.sha1, nil + return "", hash.ErrUnsupported } // Size returns the size of an object in bytes @@ -893,6 +917,11 @@ func (o *Object) setMetaData(info *api.Prop) (err error) { o.hasMetaData = true o.size = info.Size o.modTime = time.Time(info.Modified) + if o.fs.hasChecksums { + hashes := info.Hashes() + o.sha1 = hashes[hash.SHA1] + o.md5 = hashes[hash.MD5] + } return nil } @@ -972,9 +1001,21 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365 ContentType: fs.MimeType(src), } - if o.fs.useOCMtime { - opts.ExtraHeaders = map[string]string{ - "X-OC-Mtime": fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9), + if o.fs.useOCMtime || o.fs.hasChecksums { + opts.ExtraHeaders = map[string]string{} + if o.fs.useOCMtime { + opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9) + } + if o.fs.hasChecksums { + // Set an upload checksum - prefer SHA1 + // + // This is used as an upload integrity test. If we set + // only SHA1 here, owncloud will calculate the MD5 too. + if sha1, _ := src.Hash(hash.SHA1); sha1 != "" { + opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1 + } else if md5, _ := src.Hash(hash.MD5); md5 != "" { + opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5 + } } } err = o.fs.pacer.CallNoRetry(func() (bool, error) { diff --git a/docs/content/overview.md b/docs/content/overview.md index 6da854590..20ee72c88 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -36,7 +36,7 @@ Here is an overview of the major features of each cloud storage system. | pCloud | MD5, SHA1 | Yes | No | No | W | | QingStor | MD5 | No | No | No | R/W | | SFTP | MD5, SHA1 ‡ | Yes | Depends | No | - | -| WebDAV | - | Yes †† | Depends | No | - | +| WebDAV | MD5, SHA1 ††| Yes ††† | Depends | No | - | | Yandex Disk | MD5 | Yes | No | No | R/W | | The local filesystem | All | Yes | Depends | No | - | @@ -57,7 +57,9 @@ This is an SHA256 sum of all the 4MB block SHA256s. ‡ SFTP supports checksums if the same login has shell access and `md5sum` or `sha1sum` as well as `echo` are in the remote's PATH. -†† WebDAV supports modtimes when used with Owncloud and Nextcloud only. +†† WebDAV supports hashes when used with Owncloud and Nextcloud only. + +††† WebDAV supports modtimes when used with Owncloud and Nextcloud only. ‡‡ Microsoft OneDrive Personal supports SHA1 hashes, whereas OneDrive for business and SharePoint server support Microsoft's own diff --git a/docs/content/webdav.md b/docs/content/webdav.md index 718c76e8f..6662aabe3 100644 --- a/docs/content/webdav.md +++ b/docs/content/webdav.md @@ -99,7 +99,11 @@ To copy a local directory to an WebDAV directory called backup Plain WebDAV does not support modified times. However when used with Owncloud or Nextcloud rclone will support modified times. -Hashes are not supported. +Likewise plain WebDAV does not support hashes, however when used with +Owncloud or Nexcloud rclone will support SHA1 and MD5 hashes. +Depending on the exact version of Owncloud or Nextcloud hashes may +appear on all objects, or only on objects which had a hash uploaded +with them. ### Standard Options