s3: add specific provider for Storj Shared gateways

- unsupported features (Copy) are turned off for Storj
- enable urlEncodedListing for Storj provider
- set chunksize to 64Mb

backend/s3/s3.go

@@ -102,6 +102,9 @@ func init() {
 			}, {
 				Value: "StackPath",
 				Help:  "StackPath Object Storage",
+			}, {
+				Value: "Storj",
+				Help:  "Storj (S3 Compatible Gateway)",
 			}, {
 				Value: "TencentCOS",
 				Help:  "Tencent Cloud Object Storage (COS)",
@@ -288,7 +291,7 @@ func init() {
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,RackCorp,Scaleway,TencentCOS",
+			Provider: "!AWS,Alibaba,RackCorp,Scaleway,Storj,TencentCOS",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -597,6 +600,20 @@ func init() {
 				Value: "s3.eu-central-1.stackpathstorage.com",
 				Help:  "EU Endpoint",
 			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint of the Shared Gateway.",
+			Provider: "Storj",
+			Examples: []fs.OptionExample{{
+				Value: "gateway.eu1.storjshare.io",
+				Help:  "EU1 Shared Gateway",
+			}, {
+				Value: "gateway.us1.storjshare.io",
+				Help:  "US1 Shared Gateway",
+			}, {
+				Value: "gateway.ap1.storjshare.io",
+				Help:  "Asia-Pacific Shared Gateway",
+			}},
 		}, {
 			// cos endpoints: https://intl.cloud.tencent.com/document/product/436/6224
 			Name:     "endpoint",
@@ -726,7 +743,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,RackCorp",
+			Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,Storj,RackCorp",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -1014,7 +1031,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,TencentCOS",
+			Provider: "!AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1025,6 +1042,7 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview
 
 Note that this ACL is applied when server-side copying objects as S3
 doesn't copy the ACL from the source but rather writes a fresh one.`,
+			Provider: "!Storj",
 			Examples: []fs.OptionExample{{
 				Value:    "default",
 				Help:     "Owner gets Full_CONTROL.\nNo one else has access rights (default).",
@@ -1936,6 +1954,11 @@ func setQuirks(opt *Options) {
 		listObjectsV2 = false // untested
 		virtualHostStyle = false
 		urlEncodeListings = false
+	case "Storj":
+		// Force chunk size to >= 64 MiB
+		if opt.ChunkSize < 64*fs.Mebi {
+			opt.ChunkSize = 64 * fs.Mebi
+		}
 	case "TencentCOS":
 		listObjectsV2 = false // untested
 	case "Wasabi":
@@ -2070,6 +2093,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
 	}
+	if opt.Provider == "Storj" {
+		f.features.Copy = nil
+		f.features.SetTier = false
+		f.features.GetTier = false
+	}
 	// f.listMultipartUploads()
 	return f, nil
 }

docs/content/s3.md

@@ -19,6 +19,7 @@ The S3 backend can be used with a number of different providers:
 {{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}}
 {{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}}
 {{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
+{{< provider name="Storj" home="https://storj.io/" config="/s3/#storj" >}}
 {{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
 {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" end="true" >}}
 {{< /provider_list >}}
@@ -2821,6 +2822,83 @@ For Netease NOS configure as per the configurator `rclone config`
 setting the provider `Netease`.  This will automatically set
 `force_path_style = false` which is necessary for it to run properly.
 
+### Storj
+
+Storj is a decentralized cloud storage which can be used through its
+native protocol or an S3 compatible gateway.
+
+The S3 compatible gateway is configured using `rclone config` with a
+type of `s3` and with a provider name of `Storj`. Here is an example
+run of the configurator.
+
+```
+Type of storage to configure.
+Storage> s3
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth> 1
+Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> XXXX (as shown when creating the access grant)
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> XXXX (as shown when creating the access grant)
+Option endpoint.
+Endpoint of the Shared Gateway.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / EU1 Shared Gateway
+   \ (gateway.eu1.storjshare.io)
+ 2 / US1 Shared Gateway
+   \ (gateway.us1.storjshare.io)
+ 3 / Asia-Pacific Shared Gateway
+   \ (gateway.ap1.storjshare.io)
+endpoint> 1 (as shown when creating the access grant)
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+```
+
+Note that s3 credentials are generated when you [create an access
+grant](https://docs.storj.io/dcs/api-reference/s3-compatible-gateway#usage).
+
+
+#### Backend quirks
+
+- `--chunk-size` is forced to be 64 MiB or greater. This will use more
+  memory than the default of 5 MiB.
+- Server side copy is disabled as it isn't currently supported in the
+  gateway.
+- GetTier and SetTier are not supported.
+
+#### Comparison with the native protocol
+
+Use the [the native protocol](/tardigrade) to take advantage of
+client-side encryption as well as to achieve the best possible
+download performance. Uploads will be erasure-coded locally, thus a
+1gb upload will result in 2.68gb of data being uploaded to storage
+nodes across the network.
+
+Use this backend and the S3 compatible Hosted Gateway to increase
+upload performance and reduce the load on your systems and network.
+Uploads will be encrypted and erasure-coded server-side, thus a 1GB
+upload will result in only in 1GB of data being uploaded to storage
+nodes across the network.
+
+For more detailed comparison please check the documentation of the
+[tardigrade](/tardigrade) backend.
+
 ## Limitations
 
 `rclone about` is not supported by the S3 backend. Backends without
@@ -2830,4 +2908,3 @@ remote.
 
 See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
 See [rclone about](https://rclone.org/commands/rclone_about/)
-