webdav: add "fastmail" provider for Fastmail Files

This provider:

- supports the `X-OC-Mtime` header to set the mtime

- calculates SHA1 checksum server side and returns it as a `ME:sha1hex` prop

To differentiate the new hasMESHA1 quirk, the existing hasMD5 and hasSHA1
quirks for Owncloud have been renamed to hasOCMD5 and hasOCSHA1.

Fixes #6837

README.md

@@ -37,6 +37,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
+  * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)

backend/webdav/api/types.go

@@ -75,6 +75,7 @@ type Prop struct {
 	Size         int64     `xml:"DAV: prop>getcontentlength,omitempty"`
 	Modified     Time      `xml:"DAV: prop>getlastmodified,omitempty"`
 	Checksums    []string  `xml:"prop>checksums>checksum,omitempty"`
+	MESha1Hex    *string   `xml:"ME: prop>sha1hex,omitempty"` // Fastmail-specific sha1 checksum
 }
 
 // Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
@@ -102,9 +103,7 @@ func (p *Prop) StatusOK() bool {
 
 // Hashes returns a map of all checksums - may be nil
 func (p *Prop) Hashes() (hashes map[hash.Type]string) {
-	if len(p.Checksums) == 0 {
+	if len(p.Checksums) > 0 {
-		return nil
-	}
 		hashes = make(map[hash.Type]string)
 		for _, checksums := range p.Checksums {
 			checksums = strings.ToLower(checksums)
@@ -118,6 +117,13 @@ func (p *Prop) Hashes() (hashes map[hash.Type]string) {
 			}
 		}
 		return hashes
+	} else if p.MESha1Hex != nil {
+		hashes = make(map[hash.Type]string)
+		hashes[hash.SHA1] = *p.MESha1Hex
+		return hashes
+	} else {
+		return nil
+	}
 }
 
 // PropValue is a tagged name and value

backend/webdav/webdav.go

@@ -76,6 +76,9 @@ func init() {
 			Name: "vendor",
 			Help: "Name of the WebDAV site/service/software you are using.",
 			Examples: []fs.OptionExample{{
+				Value: "fastmail",
+				Help:  "Fastmail Files",
+			}, {
 				Value: "nextcloud",
 				Help:  "Nextcloud",
 			}, {
@@ -155,8 +158,9 @@ type Fs struct {
 	useOCMtime         bool          // set if can use X-OC-Mtime
 	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
 	checkBeforePurge   bool          // enables extra check that directory to purge really exists
-	hasMD5             bool          // set if can use owncloud style checksums for MD5
+	hasOCMD5           bool          // set if can use owncloud style checksums for MD5
-	hasSHA1            bool          // set if can use owncloud style checksums for SHA1
+	hasOCSHA1          bool          // set if can use owncloud style checksums for SHA1
+	hasMESHA1          bool          // set if can use fastmail style checksums for SHA1
 	ntlmAuthMu         sync.Mutex    // mutex to serialize NTLM auth roundtrips
 }
 
@@ -278,7 +282,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string)
 		},
 		NoRedirect: true,
 	}
-	if f.hasMD5 || f.hasSHA1 {
+	if f.hasOCMD5 || f.hasOCSHA1 {
 		opts.Body = bytes.NewBuffer(owncloudProps)
 	}
 	var result api.Multistatus
@@ -546,16 +550,21 @@ func (f *Fs) fetchAndSetBearerToken() error {
 // setQuirks adjusts the Fs for the vendor passed in
 func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
 	switch vendor {
+	case "fastmail":
+		f.canStream = true
+		f.precision = time.Second
+		f.useOCMtime = true
+		f.hasMESHA1 = true
 	case "owncloud":
 		f.canStream = true
 		f.precision = time.Second
 		f.useOCMtime = true
-		f.hasMD5 = true
+		f.hasOCMD5 = true
-		f.hasSHA1 = true
+		f.hasOCSHA1 = true
 	case "nextcloud":
 		f.precision = time.Second
 		f.useOCMtime = true
-		f.hasSHA1 = true
+		f.hasOCSHA1 = true
 	case "sharepoint":
 		// To mount sharepoint, two Cookies are required
 		// They have to be set instead of BasicAuth
@@ -667,7 +676,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
 			"Depth": depth,
 		},
 	}
-	if f.hasMD5 || f.hasSHA1 {
+	if f.hasOCMD5 || f.hasOCSHA1 {
 		opts.Body = bytes.NewBuffer(owncloudProps)
 	}
 	var result api.Multistatus
@@ -1126,10 +1135,10 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	hashes := hash.Set(hash.None)
-	if f.hasMD5 {
+	if f.hasOCMD5 {
 		hashes.Add(hash.MD5)
 	}
-	if f.hasSHA1 {
+	if f.hasOCSHA1 || f.hasMESHA1 {
 		hashes.Add(hash.SHA1)
 	}
 	return hashes
@@ -1197,10 +1206,10 @@ func (o *Object) Remote() string {
 
 // Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t == hash.MD5 && o.fs.hasMD5 {
+	if t == hash.MD5 && o.fs.hasOCMD5 {
 		return o.md5, nil
 	}
-	if t == hash.SHA1 && o.fs.hasSHA1 {
+	if t == hash.SHA1 && (o.fs.hasOCSHA1 || o.fs.hasMESHA1) {
 		return o.sha1, nil
 	}
 	return "", hash.ErrUnsupported
@@ -1222,12 +1231,12 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
 	o.hasMetaData = true
 	o.size = info.Size
 	o.modTime = time.Time(info.Modified)
-	if o.fs.hasMD5 || o.fs.hasSHA1 {
+	if o.fs.hasOCMD5 || o.fs.hasOCSHA1 || o.fs.hasMESHA1 {
 		hashes := info.Hashes()
-		if o.fs.hasSHA1 {
+		if o.fs.hasOCSHA1 || o.fs.hasMESHA1 {
 			o.sha1 = hashes[hash.SHA1]
 		}
-		if o.fs.hasMD5 {
+		if o.fs.hasOCMD5 {
 			o.md5 = hashes[hash.MD5]
 		}
 	}
@@ -1315,7 +1324,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		ContentType:   fs.MimeType(ctx, src),
 		Options:       options,
 	}
-	if o.fs.useOCMtime || o.fs.hasMD5 || o.fs.hasSHA1 {
+	if o.fs.useOCMtime || o.fs.hasOCMD5 || o.fs.hasOCSHA1 {
 		opts.ExtraHeaders = map[string]string{}
 		if o.fs.useOCMtime {
 			opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
@@ -1323,12 +1332,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		// Set one upload checksum
 		// Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5
 		// Nextcloud stores the checksum you supply (SHA1 or MD5) but only stores one
-		if o.fs.hasSHA1 {
+		if o.fs.hasOCSHA1 {
 			if sha1, _ := src.Hash(ctx, hash.SHA1); sha1 != "" {
 				opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1
 			}
 		}
-		if o.fs.hasMD5 && opts.ExtraHeaders["OC-Checksum"] == "" {
+		if o.fs.hasOCMD5 && opts.ExtraHeaders["OC-Checksum"] == "" {
 			if md5, _ := src.Hash(ctx, hash.MD5); md5 != "" {
 				opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5
 			}

docs/content/_index.md

@@ -121,6 +121,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
 {{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}}
 {{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}}
+{{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}}
 {{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}}
 {{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}}
 {{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}}

docs/content/overview.md

@@ -68,9 +68,9 @@ This is an SHA256 sum of all the 4 MiB block SHA256s.
 ² SFTP supports checksums if the same login has shell access and
 `md5sum` or `sha1sum` as well as `echo` are in the remote's PATH.
 
-³ WebDAV supports hashes when used with Owncloud and Nextcloud only.
+³ WebDAV supports hashes when used with Fastmail Files. Owncloud and Nextcloud only.
 
-⁴ WebDAV supports modtimes when used with Owncloud and Nextcloud only.
+⁴ WebDAV supports modtimes when used with Fastmail Files, Owncloud and Nextcloud only.
 
 ⁵ [QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash) is Microsoft's own hash.
 

docs/content/webdav.md

@@ -43,17 +43,19 @@ Choose a number from below, or type in your own value
 url> https://example.com/remote.php/webdav/
 Name of the WebDAV site/service/software you are using
 Choose a number from below, or type in your own value
- 1 / Nextcloud
+ 1 / Fastmail Files
-   \ "nextcloud"
+   \ (fastmail)
- 2 / Owncloud
+ 2 / Nextcloud
-   \ "owncloud"
+   \ (nextcloud)
- 3 / Sharepoint Online, authenticated by Microsoft account.
+ 3 / Owncloud
-   \ "sharepoint"
+   \ (owncloud)
- 4 / Sharepoint with NTLM authentication. Usually self-hosted or on-premises.
+ 4 / Sharepoint Online, authenticated by Microsoft account
-   \ "sharepoint-ntlm"
+   \ (sharepoint)
- 5 / Other site/service or software
+ 5 / Sharepoint with NTLM authentication, usually self-hosted or on-premises
-   \ "other"
+   \ (sharepoint-ntlm)
-vendor> 1
+ 6 / Other site/service or software
+   \ (other)
+vendor> 2
 User name
 user> user
 Password.
@@ -100,10 +102,10 @@ To copy a local directory to an WebDAV directory called backup
 ### Modified time and hashes ###
 
 Plain WebDAV does not support modified times.  However when used with
-Owncloud or Nextcloud rclone will support modified times.
+Fastmail Files, Owncloud or Nextcloud rclone will support modified times.
 
 Likewise plain WebDAV does not support hashes, however when used with
-Owncloud or Nextcloud rclone will support SHA1 and MD5 hashes.
+Fastmail Files, Owncloud or Nextcloud rclone will support SHA1 and MD5 hashes.
 Depending on the exact version of Owncloud or Nextcloud hashes may
 appear on all objects, or only on objects which had a hash uploaded
 with them.
@@ -242,6 +244,16 @@ Properties:
 
 See below for notes on specific providers.
 
+## Fastmail Files
+
+Use `https://webdav.fastmail.com/` or a subdirectory as the URL,
+and your Fastmail email `username@domain.tld` as the username.
+Follow [this documentation](https://www.fastmail.help/hc/en-us/articles/360058752854-App-passwords)
+to create an app password with access to `Files (WebDAV)` and use
+this as the password.
+
+Fastmail supports modified times using the `X-OC-Mtime` header.
+
 ### Owncloud
 
 Click on the settings cog in the bottom right of the page and this