2022-09-20 08:54:47 +02:00
// Package s3 provides an interface to Amazon S3 object storage
2013-06-27 21:13:07 +02:00
package s3
2013-01-08 19:53:35 +01:00
2022-07-28 12:49:19 +02:00
//go:generate go run gen_setfrom.go -o setfrom.go
2013-01-08 19:53:35 +01:00
import (
2019-06-17 10:34:30 +02:00
"context"
2019-12-31 00:17:06 +01:00
"crypto/md5"
2020-10-13 18:11:22 +02:00
"crypto/tls"
2018-01-06 15:30:10 +01:00
"encoding/base64"
"encoding/hex"
2022-06-18 09:29:21 +02:00
"encoding/json"
2019-09-16 21:25:55 +02:00
"encoding/xml"
2021-11-04 11:12:57 +01:00
"errors"
2013-01-08 19:53:35 +01:00
"fmt"
"io"
2023-11-14 13:39:50 +01:00
"math"
2016-02-01 14:11:27 +01:00
"net/http"
2019-07-23 13:24:10 +02:00
"net/url"
2013-01-08 19:53:35 +01:00
"path"
"regexp"
2019-12-31 00:17:06 +01:00
"sort"
2019-10-04 17:49:06 +02:00
"strconv"
2013-01-08 19:53:35 +01:00
"strings"
2019-11-06 11:41:03 +01:00
"sync"
2013-01-08 19:53:35 +01:00
"time"
2024-08-03 12:35:32 +02:00
"github.com/aws/aws-sdk-go-v2/aws"
v4signer "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
2024-08-06 11:33:17 +02:00
"github.com/aws/smithy-go/logging"
2024-08-03 12:35:32 +02:00
"github.com/aws/smithy-go/middleware"
smithyhttp "github.com/aws/smithy-go/transport/http"
2021-01-22 18:23:51 +01:00
"github.com/ncw/swift/v2"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/fs"
2022-07-26 16:03:32 +02:00
"github.com/rclone/rclone/fs/accounting"
2022-05-06 21:25:44 +02:00
"github.com/rclone/rclone/fs/chunksize"
2020-01-14 18:33:35 +01:00
"github.com/rclone/rclone/fs/config"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
2020-06-24 12:02:34 +02:00
"github.com/rclone/rclone/fs/operations"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/fs/walk"
2020-06-04 12:09:27 +02:00
"github.com/rclone/rclone/lib/atexit"
2019-08-09 12:29:36 +02:00
"github.com/rclone/rclone/lib/bucket"
2020-01-14 18:33:35 +01:00
"github.com/rclone/rclone/lib/encoder"
2023-08-15 21:38:02 +02:00
"github.com/rclone/rclone/lib/multipart"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/lib/pacer"
2023-08-24 18:15:18 +02:00
"github.com/rclone/rclone/lib/pool"
2019-12-31 00:17:06 +01:00
"github.com/rclone/rclone/lib/readers"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/lib/rest"
2022-07-25 17:06:15 +02:00
"github.com/rclone/rclone/lib/version"
2022-10-11 18:54:22 +02:00
"golang.org/x/net/http/httpguts"
2023-12-01 11:30:44 +01:00
"golang.org/x/sync/errgroup"
2014-03-15 17:06:11 +01:00
)
2013-06-27 21:13:07 +02:00
2023-09-21 12:32:42 +02:00
// The S3 providers
//
// Please keep these in alphabetical order, but with AWS first and
// Other last.
//
// NB if you add a new provider here, then add it in the setQuirks
// function and set the correct quirks. Test the quirks are correct by
// running the integration tests "go test -v -remote NewS3Provider:".
2023-09-21 13:35:23 +02:00
//
// See https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#adding-a-new-s3-provider
// for full information about how to add a new s3 provider.
2023-09-21 12:32:42 +02:00
var providerOption = fs . Option {
Name : fs . ConfigProvider ,
Help : "Choose your S3 provider." ,
Examples : [ ] fs . OptionExample { {
Value : "AWS" ,
Help : "Amazon Web Services (AWS) S3" ,
} , {
Value : "Alibaba" ,
Help : "Alibaba Cloud Object Storage System (OSS) formerly Aliyun" ,
} , {
Value : "ArvanCloud" ,
Help : "Arvan Cloud Object Storage (AOS)" ,
} , {
Value : "Ceph" ,
Help : "Ceph Object Storage" ,
} , {
Value : "ChinaMobile" ,
Help : "China Mobile Ecloud Elastic Object Storage (EOS)" ,
} , {
Value : "Cloudflare" ,
Help : "Cloudflare R2 Storage" ,
} , {
Value : "DigitalOcean" ,
Help : "DigitalOcean Spaces" ,
} , {
Value : "Dreamhost" ,
Help : "Dreamhost DreamObjects" ,
} , {
Value : "GCS" ,
Help : "Google Cloud Storage" ,
} , {
Value : "HuaweiOBS" ,
Help : "Huawei Object Storage Service" ,
} , {
Value : "IBMCOS" ,
Help : "IBM COS S3" ,
} , {
Value : "IDrive" ,
Help : "IDrive e2" ,
} , {
Value : "IONOS" ,
Help : "IONOS Cloud" ,
} , {
Value : "LyveCloud" ,
Help : "Seagate Lyve Cloud" ,
} , {
Value : "Leviia" ,
Help : "Leviia Object Storage" ,
} , {
Value : "Liara" ,
Help : "Liara Object Storage" ,
2023-09-21 13:38:06 +02:00
} , {
Value : "Linode" ,
Help : "Linode Object Storage" ,
2024-06-06 12:25:45 +02:00
} , {
Value : "Magalu" ,
Help : "Magalu Object Storage" ,
2023-09-21 12:32:42 +02:00
} , {
Value : "Minio" ,
Help : "Minio Object Storage" ,
} , {
Value : "Netease" ,
Help : "Netease Object Storage (NOS)" ,
} , {
Value : "Petabox" ,
Help : "Petabox Object Storage" ,
} , {
Value : "RackCorp" ,
Help : "RackCorp Object Storage" ,
2022-09-21 17:09:50 +02:00
} , {
Value : "Rclone" ,
Help : "Rclone S3 Server" ,
2023-09-21 12:32:42 +02:00
} , {
Value : "Scaleway" ,
Help : "Scaleway Object Storage" ,
} , {
Value : "SeaweedFS" ,
Help : "SeaweedFS S3" ,
} , {
Value : "StackPath" ,
Help : "StackPath Object Storage" ,
} , {
Value : "Storj" ,
Help : "Storj (S3 Compatible Gateway)" ,
} , {
Value : "Synology" ,
Help : "Synology C2 Object Storage" ,
} , {
Value : "TencentCOS" ,
Help : "Tencent Cloud Object Storage (COS)" ,
} , {
Value : "Wasabi" ,
Help : "Wasabi Object Storage" ,
} , {
Value : "Qiniu" ,
Help : "Qiniu Object Storage (Kodo)" ,
} , {
Value : "Other" ,
Help : "Any other S3 compatible provider" ,
} } ,
}
var providersList string
2013-06-27 21:13:07 +02:00
// Register with Fs
func init ( ) {
2023-09-21 12:32:42 +02:00
var s strings . Builder
for i , provider := range providerOption . Examples {
if provider . Value == "Other" {
_ , _ = s . WriteString ( " and others" )
} else {
if i != 0 {
_ , _ = s . WriteString ( ", " )
}
_ , _ = s . WriteString ( provider . Value )
}
}
providersList = s . String ( )
2016-02-18 12:35:25 +01:00
fs . Register ( & fs . RegInfo {
2016-02-15 19:11:53 +01:00
Name : "s3" ,
2023-09-21 12:32:42 +02:00
Description : "Amazon S3 Compliant Storage Providers including " + providersList ,
2016-02-15 19:11:53 +01:00
NewFs : NewFs ,
2020-06-24 12:02:34 +02:00
CommandHelp : commandHelp ,
2022-06-18 09:29:21 +02:00
Config : func ( ctx context . Context , name string , m configmap . Mapper , config fs . ConfigIn ) ( * fs . ConfigOut , error ) {
switch config . State {
case "" :
return nil , setEndpointValueForIDriveE2 ( m )
}
return nil , fmt . Errorf ( "unknown state %q" , config . State )
} ,
2022-05-24 13:32:39 +02:00
MetadataInfo : & fs . MetadataInfo {
System : systemMetadataInfo ,
Help : ` User metadata is stored as x-amz-meta- keys. S3 metadata keys are case insensitive and are always returned in lower case. ` ,
} ,
2023-09-21 12:32:42 +02:00
Options : [ ] fs . Option { providerOption , {
2018-05-14 19:06:57 +02:00
Name : "env_auth" ,
2021-08-16 11:30:01 +02:00
Help : "Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).\n\nOnly applies if access_key_id and secret_access_key is blank." ,
2018-05-14 19:06:57 +02:00
Default : false ,
2018-04-13 17:08:00 +02:00
Examples : [ ] fs . OptionExample { {
Value : "false" ,
2021-08-16 11:30:01 +02:00
Help : "Enter AWS credentials in the next step." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "true" ,
2021-08-16 11:30:01 +02:00
Help : "Get AWS credentials from the environment (env vars or IAM)." ,
2018-04-13 17:08:00 +02:00
} } ,
} , {
2023-07-06 18:55:53 +02:00
Name : "access_key_id" ,
Help : "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials." ,
Sensitive : true ,
2018-04-13 17:08:00 +02:00
} , {
2023-07-06 18:55:53 +02:00
Name : "secret_access_key" ,
Help : "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials." ,
Sensitive : true ,
2018-04-13 17:08:00 +02:00
} , {
2020-10-03 10:44:25 +02:00
// References:
// 1. https://docs.aws.amazon.com/general/latest/gr/rande.html
// 2. https://docs.aws.amazon.com/general/latest/gr/s3.html
2018-04-13 17:08:00 +02:00
Name : "region" ,
Help : "Region to connect to." ,
Provider : "AWS" ,
Examples : [ ] fs . OptionExample { {
Value : "us-east-1" ,
2020-10-13 23:43:00 +02:00
Help : "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia, or Pacific Northwest.\nLeave location constraint empty." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "us-east-2" ,
2021-08-16 11:30:01 +02:00
Help : "US East (Ohio) Region.\nNeeds location constraint us-east-2." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "us-west-1" ,
2021-08-16 11:30:01 +02:00
Help : "US West (Northern California) Region.\nNeeds location constraint us-west-1." ,
2020-10-03 10:44:25 +02:00
} , {
Value : "us-west-2" ,
2021-08-16 11:30:01 +02:00
Help : "US West (Oregon) Region.\nNeeds location constraint us-west-2." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ca-central-1" ,
2021-08-16 11:30:01 +02:00
Help : "Canada (Central) Region.\nNeeds location constraint ca-central-1." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "eu-west-1" ,
2021-08-16 11:30:01 +02:00
Help : "EU (Ireland) Region.\nNeeds location constraint EU or eu-west-1." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "eu-west-2" ,
2021-08-16 11:30:01 +02:00
Help : "EU (London) Region.\nNeeds location constraint eu-west-2." ,
2020-10-03 10:44:25 +02:00
} , {
Value : "eu-west-3" ,
2021-08-16 11:30:01 +02:00
Help : "EU (Paris) Region.\nNeeds location constraint eu-west-3." ,
2019-02-02 20:44:26 +01:00
} , {
Value : "eu-north-1" ,
2021-08-16 11:30:01 +02:00
Help : "EU (Stockholm) Region.\nNeeds location constraint eu-north-1." ,
2020-10-03 10:44:25 +02:00
} , {
Value : "eu-south-1" ,
2021-08-16 11:30:01 +02:00
Help : "EU (Milan) Region.\nNeeds location constraint eu-south-1." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "eu-central-1" ,
2021-08-16 11:30:01 +02:00
Help : "EU (Frankfurt) Region.\nNeeds location constraint eu-central-1." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ap-southeast-1" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Singapore) Region.\nNeeds location constraint ap-southeast-1." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ap-southeast-2" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Sydney) Region.\nNeeds location constraint ap-southeast-2." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ap-northeast-1" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Tokyo) Region.\nNeeds location constraint ap-northeast-1." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ap-northeast-2" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Seoul).\nNeeds location constraint ap-northeast-2." ,
2020-10-03 10:44:25 +02:00
} , {
Value : "ap-northeast-3" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Osaka-Local).\nNeeds location constraint ap-northeast-3." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ap-south-1" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Mumbai).\nNeeds location constraint ap-south-1." ,
2020-01-02 12:10:48 +01:00
} , {
Value : "ap-east-1" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Hong Kong) Region.\nNeeds location constraint ap-east-1." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "sa-east-1" ,
2021-08-16 11:30:01 +02:00
Help : "South America (Sao Paulo) Region.\nNeeds location constraint sa-east-1." ,
2024-04-01 19:17:16 +02:00
} , {
Value : "il-central-1" ,
Help : "Israel (Tel Aviv) Region.\nNeeds location constraint il-central-1." ,
2020-10-03 10:44:25 +02:00
} , {
Value : "me-south-1" ,
2021-08-16 11:30:01 +02:00
Help : "Middle East (Bahrain) Region.\nNeeds location constraint me-south-1." ,
2020-10-03 10:44:25 +02:00
} , {
Value : "af-south-1" ,
2021-08-16 11:30:01 +02:00
Help : "Africa (Cape Town) Region.\nNeeds location constraint af-south-1." ,
2020-10-03 10:44:25 +02:00
} , {
Value : "cn-north-1" ,
2021-08-16 11:30:01 +02:00
Help : "China (Beijing) Region.\nNeeds location constraint cn-north-1." ,
2020-10-03 10:44:25 +02:00
} , {
Value : "cn-northwest-1" ,
2021-08-16 11:30:01 +02:00
Help : "China (Ningxia) Region.\nNeeds location constraint cn-northwest-1." ,
2020-10-03 10:44:25 +02:00
} , {
Value : "us-gov-east-1" ,
2021-08-16 11:30:01 +02:00
Help : "AWS GovCloud (US-East) Region.\nNeeds location constraint us-gov-east-1." ,
2020-10-03 10:44:25 +02:00
} , {
Value : "us-gov-west-1" ,
2021-08-16 11:30:01 +02:00
Help : "AWS GovCloud (US) Region.\nNeeds location constraint us-gov-west-1." ,
2018-04-13 17:08:00 +02:00
} } ,
2021-11-09 12:46:58 +01:00
} , {
Name : "region" ,
Help : "region - the location where your bucket will be created and your data stored.\n" ,
Provider : "RackCorp" ,
Examples : [ ] fs . OptionExample { {
Value : "global" ,
Help : "Global CDN (All locations) Region" ,
} , {
Value : "au" ,
Help : "Australia (All states)" ,
} , {
Value : "au-nsw" ,
Help : "NSW (Australia) Region" ,
} , {
Value : "au-qld" ,
Help : "QLD (Australia) Region" ,
} , {
Value : "au-vic" ,
Help : "VIC (Australia) Region" ,
} , {
Value : "au-wa" ,
Help : "Perth (Australia) Region" ,
} , {
Value : "ph" ,
Help : "Manila (Philippines) Region" ,
} , {
Value : "th" ,
Help : "Bangkok (Thailand) Region" ,
} , {
Value : "hk" ,
Help : "HK (Hong Kong) Region" ,
} , {
Value : "mn" ,
Help : "Ulaanbaatar (Mongolia) Region" ,
} , {
Value : "kg" ,
Help : "Bishkek (Kyrgyzstan) Region" ,
} , {
Value : "id" ,
Help : "Jakarta (Indonesia) Region" ,
} , {
Value : "jp" ,
Help : "Tokyo (Japan) Region" ,
} , {
Value : "sg" ,
Help : "SG (Singapore) Region" ,
} , {
Value : "de" ,
Help : "Frankfurt (Germany) Region" ,
} , {
Value : "us" ,
Help : "USA (AnyCast) Region" ,
} , {
Value : "us-east-1" ,
Help : "New York (USA) Region" ,
} , {
Value : "us-west-1" ,
Help : "Freemont (USA) Region" ,
} , {
Value : "nz" ,
Help : "Auckland (New Zealand) Region" ,
} } ,
2020-06-12 17:04:16 +02:00
} , {
Name : "region" ,
Help : "Region to connect to." ,
Provider : "Scaleway" ,
Examples : [ ] fs . OptionExample { {
Value : "nl-ams" ,
Help : "Amsterdam, The Netherlands" ,
} , {
Value : "fr-par" ,
Help : "Paris, France" ,
2022-05-17 15:24:32 +02:00
} , {
Value : "pl-waw" ,
Help : "Warsaw, Poland" ,
2020-06-12 17:04:16 +02:00
} } ,
2022-06-07 09:41:46 +02:00
} , {
Name : "region" ,
Help : "Region to connect to. - the location where your bucket will be created and your data stored. Need bo be same with your endpoint.\n" ,
Provider : "HuaweiOBS" ,
Examples : [ ] fs . OptionExample { {
Value : "af-south-1" ,
Help : "AF-Johannesburg" ,
} , {
Value : "ap-southeast-2" ,
Help : "AP-Bangkok" ,
} , {
Value : "ap-southeast-3" ,
Help : "AP-Singapore" ,
} , {
Value : "cn-east-3" ,
Help : "CN East-Shanghai1" ,
} , {
Value : "cn-east-2" ,
Help : "CN East-Shanghai2" ,
} , {
Value : "cn-north-1" ,
Help : "CN North-Beijing1" ,
} , {
Value : "cn-north-4" ,
Help : "CN North-Beijing4" ,
} , {
Value : "cn-south-1" ,
Help : "CN South-Guangzhou" ,
} , {
Value : "ap-southeast-1" ,
Help : "CN-Hong Kong" ,
} , {
Value : "sa-argentina-1" ,
Help : "LA-Buenos Aires1" ,
} , {
Value : "sa-peru-1" ,
Help : "LA-Lima1" ,
} , {
Value : "na-mexico-1" ,
Help : "LA-Mexico City1" ,
} , {
Value : "sa-chile-1" ,
Help : "LA-Santiago2" ,
} , {
Value : "sa-brazil-1" ,
Help : "LA-Sao Paulo1" ,
} , {
Value : "ru-northwest-2" ,
Help : "RU-Moscow2" ,
} } ,
2022-05-06 13:09:29 +02:00
} , {
Name : "region" ,
Help : "Region to connect to." ,
Provider : "Cloudflare" ,
Examples : [ ] fs . OptionExample { {
Value : "auto" ,
Help : "R2 buckets are automatically distributed across Cloudflare's data centers for low latency." ,
} } ,
2022-10-13 16:49:22 +02:00
} , {
// References:
// https://developer.qiniu.com/kodo/4088/s3-access-domainname
Name : "region" ,
Help : "Region to connect to." ,
Provider : "Qiniu" ,
Examples : [ ] fs . OptionExample { {
Value : "cn-east-1" ,
Help : "The default endpoint - a good choice if you are unsure.\nEast China Region 1.\nNeeds location constraint cn-east-1." ,
} , {
Value : "cn-east-2" ,
Help : "East China Region 2.\nNeeds location constraint cn-east-2." ,
} , {
Value : "cn-north-1" ,
Help : "North China Region 1.\nNeeds location constraint cn-north-1." ,
} , {
Value : "cn-south-1" ,
Help : "South China Region 1.\nNeeds location constraint cn-south-1." ,
} , {
Value : "us-north-1" ,
Help : "North America Region.\nNeeds location constraint us-north-1." ,
} , {
Value : "ap-southeast-1" ,
Help : "Southeast Asia Region 1.\nNeeds location constraint ap-southeast-1." ,
} , {
Value : "ap-northeast-1" ,
Help : "Northeast Asia Region 1.\nNeeds location constraint ap-northeast-1." ,
} } ,
2022-09-15 17:04:34 +02:00
} , {
Name : "region" ,
Help : "Region where your bucket will be created and your data stored.\n" ,
Provider : "IONOS" ,
Examples : [ ] fs . OptionExample { {
Value : "de" ,
Help : "Frankfurt, Germany" ,
} , {
Value : "eu-central-2" ,
Help : "Berlin, Germany" ,
} , {
Value : "eu-south-2" ,
Help : "Logrono, Spain" ,
} } ,
2023-05-05 10:44:25 +02:00
} , {
Name : "region" ,
Help : "Region where your bucket will be created and your data stored.\n" ,
Provider : "Petabox" ,
Examples : [ ] fs . OptionExample { {
Value : "us-east-1" ,
Help : "US East (N. Virginia)" ,
} , {
Value : "eu-central-1" ,
Help : "Europe (Frankfurt)" ,
} , {
Value : "ap-southeast-1" ,
Help : "Asia Pacific (Singapore)" ,
} , {
Value : "me-south-1" ,
Help : "Middle East (Bahrain)" ,
} , {
Value : "sa-east-1" ,
Help : "South America (São Paulo)" ,
} } ,
2023-07-06 11:54:07 +02:00
} , {
Name : "region" ,
Help : "Region where your data stored.\n" ,
Provider : "Synology" ,
Examples : [ ] fs . OptionExample { {
Value : "eu-001" ,
Help : "Europe Region 1" ,
} , {
Value : "eu-002" ,
Help : "Europe Region 2" ,
} , {
Value : "us-001" ,
Help : "US Region 1" ,
} , {
Value : "us-002" ,
Help : "US Region 2" ,
} , {
Value : "tw-001" ,
Help : "Asia (Taiwan)" ,
} } ,
2018-04-13 17:08:00 +02:00
} , {
Name : "region" ,
2021-08-16 11:30:01 +02:00
Help : "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region." ,
2024-06-06 12:25:45 +02:00
Provider : "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Storj,Synology,TencentCOS,HuaweiOBS,IDrive" ,
2018-04-13 17:08:00 +02:00
Examples : [ ] fs . OptionExample { {
Value : "" ,
2021-08-16 11:30:01 +02:00
Help : "Use this if unsure.\nWill use v4 signatures and an empty region." ,
2018-04-13 17:08:00 +02:00
} , {
Value : "other-v2-signature" ,
2021-08-16 11:30:01 +02:00
Help : "Use this only if v4 signatures don't work.\nE.g. pre Jewel/v10 CEPH." ,
2018-04-13 17:08:00 +02:00
} } ,
} , {
Name : "endpoint" ,
2021-08-16 11:30:01 +02:00
Help : "Endpoint for S3 API.\n\nLeave blank if using AWS to use the default endpoint for the region." ,
2018-04-13 17:08:00 +02:00
Provider : "AWS" ,
2022-03-24 12:57:00 +01:00
} , {
// ChinaMobile endpoints: https://ecloud.10086.cn/op-help-center/doc/article/24534
Name : "endpoint" ,
Help : "Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API." ,
Provider : "ChinaMobile" ,
Examples : [ ] fs . OptionExample { {
Value : "eos-wuxi-1.cmecloud.cn" ,
Help : "The default endpoint - a good choice if you are unsure.\nEast China (Suzhou)" ,
} , {
Value : "eos-jinan-1.cmecloud.cn" ,
Help : "East China (Jinan)" ,
} , {
Value : "eos-ningbo-1.cmecloud.cn" ,
Help : "East China (Hangzhou)" ,
} , {
Value : "eos-shanghai-1.cmecloud.cn" ,
Help : "East China (Shanghai-1)" ,
} , {
Value : "eos-zhengzhou-1.cmecloud.cn" ,
Help : "Central China (Zhengzhou)" ,
} , {
Value : "eos-hunan-1.cmecloud.cn" ,
Help : "Central China (Changsha-1)" ,
} , {
Value : "eos-zhuzhou-1.cmecloud.cn" ,
Help : "Central China (Changsha-2)" ,
} , {
Value : "eos-guangzhou-1.cmecloud.cn" ,
Help : "South China (Guangzhou-2)" ,
} , {
Value : "eos-dongguan-1.cmecloud.cn" ,
Help : "South China (Guangzhou-3)" ,
} , {
Value : "eos-beijing-1.cmecloud.cn" ,
Help : "North China (Beijing-1)" ,
} , {
Value : "eos-beijing-2.cmecloud.cn" ,
Help : "North China (Beijing-2)" ,
} , {
Value : "eos-beijing-4.cmecloud.cn" ,
Help : "North China (Beijing-3)" ,
} , {
Value : "eos-huhehaote-1.cmecloud.cn" ,
Help : "North China (Huhehaote)" ,
} , {
Value : "eos-chengdu-1.cmecloud.cn" ,
Help : "Southwest China (Chengdu)" ,
} , {
Value : "eos-chongqing-1.cmecloud.cn" ,
Help : "Southwest China (Chongqing)" ,
} , {
Value : "eos-guiyang-1.cmecloud.cn" ,
Help : "Southwest China (Guiyang)" ,
} , {
Value : "eos-xian-1.cmecloud.cn" ,
Help : "Nouthwest China (Xian)" ,
} , {
Value : "eos-yunnan.cmecloud.cn" ,
Help : "Yunnan China (Kunming)" ,
} , {
Value : "eos-yunnan-2.cmecloud.cn" ,
Help : "Yunnan China (Kunming-2)" ,
} , {
Value : "eos-tianjin-1.cmecloud.cn" ,
Help : "Tianjin China (Tianjin)" ,
} , {
Value : "eos-jilin-1.cmecloud.cn" ,
Help : "Jilin China (Changchun)" ,
} , {
Value : "eos-hubei-1.cmecloud.cn" ,
Help : "Hubei China (Xiangyan)" ,
} , {
Value : "eos-jiangxi-1.cmecloud.cn" ,
Help : "Jiangxi China (Nanchang)" ,
} , {
Value : "eos-gansu-1.cmecloud.cn" ,
Help : "Gansu China (Lanzhou)" ,
} , {
Value : "eos-shanxi-1.cmecloud.cn" ,
Help : "Shanxi China (Taiyuan)" ,
} , {
Value : "eos-liaoning-1.cmecloud.cn" ,
Help : "Liaoning China (Shenyang)" ,
} , {
Value : "eos-hebei-1.cmecloud.cn" ,
Help : "Hebei China (Shijiazhuang)" ,
} , {
Value : "eos-fujian-1.cmecloud.cn" ,
Help : "Fujian China (Xiamen)" ,
} , {
Value : "eos-guangxi-1.cmecloud.cn" ,
Help : "Guangxi China (Nanning)" ,
} , {
Value : "eos-anhui-1.cmecloud.cn" ,
Help : "Anhui China (Huainan)" ,
} } ,
2022-04-28 11:42:30 +02:00
} , {
2023-06-17 13:47:23 +02:00
// ArvanCloud endpoints: https://www.arvancloud.ir/en/products/cloud-storage
2022-04-28 11:42:30 +02:00
Name : "endpoint" ,
Help : "Endpoint for Arvan Cloud Object Storage (AOS) API." ,
Provider : "ArvanCloud" ,
Examples : [ ] fs . OptionExample { {
2023-06-17 13:47:23 +02:00
Value : "s3.ir-thr-at1.arvanstorage.ir" ,
Help : "The default endpoint - a good choice if you are unsure.\nTehran Iran (Simin)" ,
2022-04-28 11:42:30 +02:00
} , {
2023-06-17 13:47:23 +02:00
Value : "s3.ir-tbz-sh1.arvanstorage.ir" ,
2022-04-28 11:42:30 +02:00
Help : "Tabriz Iran (Shahriar)" ,
} } ,
2018-04-13 17:08:00 +02:00
} , {
Name : "endpoint" ,
2021-08-16 11:30:01 +02:00
Help : "Endpoint for IBM COS S3 API.\n\nSpecify if using an IBM COS On Premise." ,
2018-04-13 17:08:00 +02:00
Provider : "IBMCOS" ,
Examples : [ ] fs . OptionExample { {
2020-08-30 18:21:11 +02:00
Value : "s3.us.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Cross Region Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.dal.us.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Cross Region Dallas Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.wdc.us.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Cross Region Washington DC Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.sjc.us.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Cross Region San Jose Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.us.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Cross Region Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.dal.us.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Cross Region Dallas Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.wdc.us.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Cross Region Washington DC Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.sjc.us.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Cross Region San Jose Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.us-east.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Region East Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.us-east.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Region East Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.us-south.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Region South Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.us-south.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "US Region South Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.eu.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "EU Cross Region Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.fra.eu.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "EU Cross Region Frankfurt Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.mil.eu.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "EU Cross Region Milan Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.ams.eu.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "EU Cross Region Amsterdam Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.eu.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "EU Cross Region Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.fra.eu.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "EU Cross Region Frankfurt Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.mil.eu.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "EU Cross Region Milan Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.ams.eu.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "EU Cross Region Amsterdam Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.eu-gb.cloud-object-storage.appdomain.cloud" ,
2019-02-07 18:41:17 +01:00
Help : "Great Britain Endpoint" ,
2018-04-13 17:08:00 +02:00
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.eu-gb.cloud-object-storage.appdomain.cloud" ,
2019-02-07 18:41:17 +01:00
Help : "Great Britain Private Endpoint" ,
2018-04-13 17:08:00 +02:00
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.eu-de.cloud-object-storage.appdomain.cloud" ,
Help : "EU Region DE Endpoint" ,
} , {
Value : "s3.private.eu-de.cloud-object-storage.appdomain.cloud" ,
Help : "EU Region DE Private Endpoint" ,
} , {
Value : "s3.ap.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "APAC Cross Regional Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.tok.ap.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "APAC Cross Regional Tokyo Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.hkg.ap.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "APAC Cross Regional HongKong Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.seo.ap.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "APAC Cross Regional Seoul Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.ap.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "APAC Cross Regional Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.tok.ap.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "APAC Cross Regional Tokyo Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.hkg.ap.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "APAC Cross Regional HongKong Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.seo.ap.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "APAC Cross Regional Seoul Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.jp-tok.cloud-object-storage.appdomain.cloud" ,
Help : "APAC Region Japan Endpoint" ,
} , {
Value : "s3.private.jp-tok.cloud-object-storage.appdomain.cloud" ,
Help : "APAC Region Japan Private Endpoint" ,
} , {
Value : "s3.au-syd.cloud-object-storage.appdomain.cloud" ,
Help : "APAC Region Australia Endpoint" ,
} , {
Value : "s3.private.au-syd.cloud-object-storage.appdomain.cloud" ,
Help : "APAC Region Australia Private Endpoint" ,
} , {
Value : "s3.ams03.cloud-object-storage.appdomain.cloud" ,
Help : "Amsterdam Single Site Endpoint" ,
} , {
Value : "s3.private.ams03.cloud-object-storage.appdomain.cloud" ,
Help : "Amsterdam Single Site Private Endpoint" ,
} , {
Value : "s3.che01.cloud-object-storage.appdomain.cloud" ,
Help : "Chennai Single Site Endpoint" ,
} , {
Value : "s3.private.che01.cloud-object-storage.appdomain.cloud" ,
Help : "Chennai Single Site Private Endpoint" ,
} , {
Value : "s3.mel01.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "Melbourne Single Site Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.mel01.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "Melbourne Single Site Private Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.osl01.cloud-object-storage.appdomain.cloud" ,
Help : "Oslo Single Site Endpoint" ,
} , {
Value : "s3.private.osl01.cloud-object-storage.appdomain.cloud" ,
Help : "Oslo Single Site Private Endpoint" ,
} , {
Value : "s3.tor01.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "Toronto Single Site Endpoint" ,
} , {
2020-08-30 18:21:11 +02:00
Value : "s3.private.tor01.cloud-object-storage.appdomain.cloud" ,
2018-04-13 17:08:00 +02:00
Help : "Toronto Single Site Private Endpoint" ,
2020-08-30 18:21:11 +02:00
} , {
Value : "s3.seo01.cloud-object-storage.appdomain.cloud" ,
Help : "Seoul Single Site Endpoint" ,
} , {
Value : "s3.private.seo01.cloud-object-storage.appdomain.cloud" ,
Help : "Seoul Single Site Private Endpoint" ,
} , {
Value : "s3.mon01.cloud-object-storage.appdomain.cloud" ,
Help : "Montreal Single Site Endpoint" ,
} , {
Value : "s3.private.mon01.cloud-object-storage.appdomain.cloud" ,
Help : "Montreal Single Site Private Endpoint" ,
} , {
Value : "s3.mex01.cloud-object-storage.appdomain.cloud" ,
Help : "Mexico Single Site Endpoint" ,
} , {
Value : "s3.private.mex01.cloud-object-storage.appdomain.cloud" ,
Help : "Mexico Single Site Private Endpoint" ,
} , {
Value : "s3.sjc04.cloud-object-storage.appdomain.cloud" ,
Help : "San Jose Single Site Endpoint" ,
} , {
Value : "s3.private.sjc04.cloud-object-storage.appdomain.cloud" ,
Help : "San Jose Single Site Private Endpoint" ,
} , {
Value : "s3.mil01.cloud-object-storage.appdomain.cloud" ,
Help : "Milan Single Site Endpoint" ,
} , {
Value : "s3.private.mil01.cloud-object-storage.appdomain.cloud" ,
Help : "Milan Single Site Private Endpoint" ,
} , {
Value : "s3.hkg02.cloud-object-storage.appdomain.cloud" ,
Help : "Hong Kong Single Site Endpoint" ,
} , {
Value : "s3.private.hkg02.cloud-object-storage.appdomain.cloud" ,
Help : "Hong Kong Single Site Private Endpoint" ,
} , {
Value : "s3.par01.cloud-object-storage.appdomain.cloud" ,
Help : "Paris Single Site Endpoint" ,
} , {
Value : "s3.private.par01.cloud-object-storage.appdomain.cloud" ,
Help : "Paris Single Site Private Endpoint" ,
} , {
Value : "s3.sng01.cloud-object-storage.appdomain.cloud" ,
Help : "Singapore Single Site Endpoint" ,
} , {
Value : "s3.private.sng01.cloud-object-storage.appdomain.cloud" ,
Help : "Singapore Single Site Private Endpoint" ,
2018-04-13 17:08:00 +02:00
} } ,
2022-09-15 17:04:34 +02:00
} , {
Name : "endpoint" ,
Help : "Endpoint for IONOS S3 Object Storage.\n\nSpecify the endpoint from the same region." ,
Provider : "IONOS" ,
Examples : [ ] fs . OptionExample { {
Value : "s3-eu-central-1.ionoscloud.com" ,
Help : "Frankfurt, Germany" ,
} , {
Value : "s3-eu-central-2.ionoscloud.com" ,
Help : "Berlin, Germany" ,
} , {
Value : "s3-eu-south-2.ionoscloud.com" ,
Help : "Logrono, Spain" ,
} } ,
2023-05-05 10:44:25 +02:00
} , {
Name : "endpoint" ,
Help : "Endpoint for Petabox S3 Object Storage.\n\nSpecify the endpoint from the same region." ,
Provider : "Petabox" ,
Required : true ,
Examples : [ ] fs . OptionExample { {
Value : "s3.petabox.io" ,
Help : "US East (N. Virginia)" ,
} , {
Value : "s3.us-east-1.petabox.io" ,
Help : "US East (N. Virginia)" ,
} , {
Value : "s3.eu-central-1.petabox.io" ,
Help : "Europe (Frankfurt)" ,
} , {
Value : "s3.ap-southeast-1.petabox.io" ,
Help : "Asia Pacific (Singapore)" ,
} , {
Value : "s3.me-south-1.petabox.io" ,
Help : "Middle East (Bahrain)" ,
} , {
Value : "s3.sa-east-1.petabox.io" ,
Help : "South America (São Paulo)" ,
} } ,
2023-07-16 19:08:47 +02:00
} , {
// Leviia endpoints: https://www.leviia.com/object-storage/
Name : "endpoint" ,
Help : "Endpoint for Leviia Object Storage API." ,
Provider : "Leviia" ,
Examples : [ ] fs . OptionExample { {
Value : "s3.leviia.com" ,
Help : "The default endpoint\nLeviia" ,
} } ,
2022-12-06 13:25:23 +01:00
} , {
// Liara endpoints: https://liara.ir/landing/object-storage
Name : "endpoint" ,
Help : "Endpoint for Liara Object Storage API." ,
Provider : "Liara" ,
Examples : [ ] fs . OptionExample { {
Value : "storage.iran.liara.space" ,
Help : "The default endpoint\nIran" ,
} } ,
2023-09-21 13:38:06 +02:00
} , {
// Linode endpoints: https://www.linode.com/docs/products/storage/object-storage/guides/urls/#cluster-url-s3-endpoint
Name : "endpoint" ,
Help : "Endpoint for Linode Object Storage API." ,
Provider : "Linode" ,
Examples : [ ] fs . OptionExample { {
Value : "us-southeast-1.linodeobjects.com" ,
Help : "Atlanta, GA (USA), us-southeast-1" ,
} , {
Value : "us-ord-1.linodeobjects.com" ,
Help : "Chicago, IL (USA), us-ord-1" ,
} , {
Value : "eu-central-1.linodeobjects.com" ,
Help : "Frankfurt (Germany), eu-central-1" ,
} , {
Value : "it-mil-1.linodeobjects.com" ,
Help : "Milan (Italy), it-mil-1" ,
} , {
Value : "us-east-1.linodeobjects.com" ,
Help : "Newark, NJ (USA), us-east-1" ,
} , {
Value : "fr-par-1.linodeobjects.com" ,
Help : "Paris (France), fr-par-1" ,
} , {
Value : "us-sea-1.linodeobjects.com" ,
Help : "Seattle, WA (USA), us-sea-1" ,
} , {
Value : "ap-south-1.linodeobjects.com" ,
Help : "Singapore ap-south-1" ,
} , {
Value : "se-sto-1.linodeobjects.com" ,
Help : "Stockholm (Sweden), se-sto-1" ,
} , {
Value : "us-iad-1.linodeobjects.com" ,
Help : "Washington, DC, (USA), us-iad-1" ,
} } ,
2024-06-06 12:25:45 +02:00
} , {
// Magalu endpoints: https://docs.magalu.cloud/docs/object-storage/how-to/copy-url
Name : "endpoint" ,
Help : "Endpoint for Magalu Object Storage API." ,
Provider : "Magalu" ,
Examples : [ ] fs . OptionExample { {
Value : "br-se1.magaluobjects.com" ,
Help : "São Paulo, SP (BR), br-se1" ,
} , {
Value : "br-ne1.magaluobjects.com" ,
Help : "Fortaleza, CE (BR), br-ne1" ,
} ,
} ,
2019-01-12 17:46:45 +01:00
} , {
// oss endpoints: https://help.aliyun.com/document_detail/31837.html
Name : "endpoint" ,
Help : "Endpoint for OSS API." ,
Provider : "Alibaba" ,
Examples : [ ] fs . OptionExample { {
2021-07-08 13:03:04 +02:00
Value : "oss-accelerate.aliyuncs.com" ,
Help : "Global Accelerate" ,
} , {
Value : "oss-accelerate-overseas.aliyuncs.com" ,
Help : "Global Accelerate (outside mainland China)" ,
} , {
2019-01-12 17:46:45 +01:00
Value : "oss-cn-hangzhou.aliyuncs.com" ,
Help : "East China 1 (Hangzhou)" ,
} , {
Value : "oss-cn-shanghai.aliyuncs.com" ,
Help : "East China 2 (Shanghai)" ,
} , {
Value : "oss-cn-qingdao.aliyuncs.com" ,
Help : "North China 1 (Qingdao)" ,
} , {
Value : "oss-cn-beijing.aliyuncs.com" ,
Help : "North China 2 (Beijing)" ,
} , {
Value : "oss-cn-zhangjiakou.aliyuncs.com" ,
Help : "North China 3 (Zhangjiakou)" ,
} , {
Value : "oss-cn-huhehaote.aliyuncs.com" ,
2021-07-08 13:03:04 +02:00
Help : "North China 5 (Hohhot)" ,
} , {
Value : "oss-cn-wulanchabu.aliyuncs.com" ,
Help : "North China 6 (Ulanqab)" ,
2019-01-12 17:46:45 +01:00
} , {
Value : "oss-cn-shenzhen.aliyuncs.com" ,
Help : "South China 1 (Shenzhen)" ,
2021-07-08 13:03:04 +02:00
} , {
Value : "oss-cn-heyuan.aliyuncs.com" ,
Help : "South China 2 (Heyuan)" ,
} , {
Value : "oss-cn-guangzhou.aliyuncs.com" ,
Help : "South China 3 (Guangzhou)" ,
} , {
Value : "oss-cn-chengdu.aliyuncs.com" ,
Help : "West China 1 (Chengdu)" ,
2019-01-12 17:46:45 +01:00
} , {
Value : "oss-cn-hongkong.aliyuncs.com" ,
Help : "Hong Kong (Hong Kong)" ,
} , {
Value : "oss-us-west-1.aliyuncs.com" ,
Help : "US West 1 (Silicon Valley)" ,
} , {
Value : "oss-us-east-1.aliyuncs.com" ,
Help : "US East 1 (Virginia)" ,
} , {
Value : "oss-ap-southeast-1.aliyuncs.com" ,
Help : "Southeast Asia Southeast 1 (Singapore)" ,
} , {
Value : "oss-ap-southeast-2.aliyuncs.com" ,
Help : "Asia Pacific Southeast 2 (Sydney)" ,
} , {
Value : "oss-ap-southeast-3.aliyuncs.com" ,
Help : "Southeast Asia Southeast 3 (Kuala Lumpur)" ,
} , {
Value : "oss-ap-southeast-5.aliyuncs.com" ,
Help : "Asia Pacific Southeast 5 (Jakarta)" ,
} , {
Value : "oss-ap-northeast-1.aliyuncs.com" ,
Help : "Asia Pacific Northeast 1 (Japan)" ,
} , {
Value : "oss-ap-south-1.aliyuncs.com" ,
Help : "Asia Pacific South 1 (Mumbai)" ,
} , {
Value : "oss-eu-central-1.aliyuncs.com" ,
Help : "Central Europe 1 (Frankfurt)" ,
} , {
Value : "oss-eu-west-1.aliyuncs.com" ,
Help : "West Europe (London)" ,
} , {
Value : "oss-me-east-1.aliyuncs.com" ,
Help : "Middle East 1 (Dubai)" ,
} } ,
2022-06-07 09:41:46 +02:00
} , {
// obs endpoints: https://developer.huaweicloud.com/intl/en-us/endpoint?OBS
Name : "endpoint" ,
Help : "Endpoint for OBS API." ,
Provider : "HuaweiOBS" ,
Examples : [ ] fs . OptionExample { {
Value : "obs.af-south-1.myhuaweicloud.com" ,
Help : "AF-Johannesburg" ,
} , {
Value : "obs.ap-southeast-2.myhuaweicloud.com" ,
Help : "AP-Bangkok" ,
} , {
Value : "obs.ap-southeast-3.myhuaweicloud.com" ,
Help : "AP-Singapore" ,
} , {
Value : "obs.cn-east-3.myhuaweicloud.com" ,
Help : "CN East-Shanghai1" ,
} , {
Value : "obs.cn-east-2.myhuaweicloud.com" ,
Help : "CN East-Shanghai2" ,
} , {
Value : "obs.cn-north-1.myhuaweicloud.com" ,
Help : "CN North-Beijing1" ,
} , {
Value : "obs.cn-north-4.myhuaweicloud.com" ,
Help : "CN North-Beijing4" ,
} , {
Value : "obs.cn-south-1.myhuaweicloud.com" ,
Help : "CN South-Guangzhou" ,
} , {
Value : "obs.ap-southeast-1.myhuaweicloud.com" ,
Help : "CN-Hong Kong" ,
} , {
Value : "obs.sa-argentina-1.myhuaweicloud.com" ,
Help : "LA-Buenos Aires1" ,
} , {
Value : "obs.sa-peru-1.myhuaweicloud.com" ,
Help : "LA-Lima1" ,
} , {
Value : "obs.na-mexico-1.myhuaweicloud.com" ,
Help : "LA-Mexico City1" ,
} , {
Value : "obs.sa-chile-1.myhuaweicloud.com" ,
Help : "LA-Santiago2" ,
} , {
Value : "obs.sa-brazil-1.myhuaweicloud.com" ,
Help : "LA-Sao Paulo1" ,
} , {
Value : "obs.ru-northwest-2.myhuaweicloud.com" ,
Help : "RU-Moscow2" ,
} } ,
2020-06-12 17:04:16 +02:00
} , {
Name : "endpoint" ,
Help : "Endpoint for Scaleway Object Storage." ,
Provider : "Scaleway" ,
Examples : [ ] fs . OptionExample { {
Value : "s3.nl-ams.scw.cloud" ,
Help : "Amsterdam Endpoint" ,
} , {
Value : "s3.fr-par.scw.cloud" ,
Help : "Paris Endpoint" ,
2022-05-17 15:24:32 +02:00
} , {
Value : "s3.pl-waw.scw.cloud" ,
Help : "Warsaw Endpoint" ,
2020-06-12 17:04:16 +02:00
} } ,
2020-01-31 00:21:24 +01:00
} , {
Name : "endpoint" ,
Help : "Endpoint for StackPath Object Storage." ,
Provider : "StackPath" ,
Examples : [ ] fs . OptionExample { {
Value : "s3.us-east-2.stackpathstorage.com" ,
Help : "US East Endpoint" ,
} , {
Value : "s3.us-west-1.stackpathstorage.com" ,
Help : "US West Endpoint" ,
} , {
Value : "s3.eu-central-1.stackpathstorage.com" ,
Help : "EU Endpoint" ,
} } ,
2023-01-09 07:54:51 +01:00
} , {
Name : "endpoint" ,
Help : "Endpoint for Google Cloud Storage." ,
Provider : "GCS" ,
Examples : [ ] fs . OptionExample { {
Value : "https://storage.googleapis.com" ,
Help : "Google Cloud Storage endpoint" ,
} } ,
2021-09-29 15:43:06 +02:00
} , {
Name : "endpoint" ,
2022-12-22 14:44:10 +01:00
Help : "Endpoint for Storj Gateway." ,
2021-09-29 15:43:06 +02:00
Provider : "Storj" ,
Examples : [ ] fs . OptionExample { {
2022-12-22 14:44:10 +01:00
Value : "gateway.storjshare.io" ,
Help : "Global Hosted Gateway" ,
2021-09-29 15:43:06 +02:00
} } ,
2023-07-06 11:54:07 +02:00
} , {
Name : "endpoint" ,
Help : "Endpoint for Synology C2 Object Storage API." ,
Provider : "Synology" ,
Examples : [ ] fs . OptionExample { {
Value : "eu-001.s3.synologyc2.net" ,
Help : "EU Endpoint 1" ,
} , {
Value : "eu-002.s3.synologyc2.net" ,
Help : "EU Endpoint 2" ,
} , {
Value : "us-001.s3.synologyc2.net" ,
Help : "US Endpoint 1" ,
} , {
Value : "us-002.s3.synologyc2.net" ,
Help : "US Endpoint 2" ,
} , {
Value : "tw-001.s3.synologyc2.net" ,
Help : "TW Endpoint 1" ,
} } ,
2020-09-08 17:34:25 +02:00
} , {
// cos endpoints: https://intl.cloud.tencent.com/document/product/436/6224
Name : "endpoint" ,
Help : "Endpoint for Tencent COS API." ,
Provider : "TencentCOS" ,
Examples : [ ] fs . OptionExample { {
Value : "cos.ap-beijing.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Beijing Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-nanjing.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Nanjing Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-shanghai.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Shanghai Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-guangzhou.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Guangzhou Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-nanjing.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Nanjing Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-chengdu.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Chengdu Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-chongqing.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Chongqing Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-hongkong.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Hong Kong (China) Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-singapore.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Singapore Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-mumbai.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Mumbai Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-seoul.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Seoul Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-bangkok.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Bangkok Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.ap-tokyo.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Tokyo Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.na-siliconvalley.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Silicon Valley Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.na-ashburn.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Virginia Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.na-toronto.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Toronto Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.eu-frankfurt.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Frankfurt Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.eu-moscow.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Moscow Region" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "cos.accelerate.myqcloud.com" ,
2021-08-16 11:30:01 +02:00
Help : "Use Tencent COS Accelerate Endpoint" ,
2020-09-08 17:34:25 +02:00
} } ,
2021-11-09 12:46:58 +01:00
} , {
// RackCorp endpoints: https://www.rackcorp.com/storage/s3storage
Name : "endpoint" ,
Help : "Endpoint for RackCorp Object Storage." ,
Provider : "RackCorp" ,
Examples : [ ] fs . OptionExample { {
Value : "s3.rackcorp.com" ,
Help : "Global (AnyCast) Endpoint" ,
} , {
Value : "au.s3.rackcorp.com" ,
Help : "Australia (Anycast) Endpoint" ,
} , {
Value : "au-nsw.s3.rackcorp.com" ,
Help : "Sydney (Australia) Endpoint" ,
} , {
Value : "au-qld.s3.rackcorp.com" ,
Help : "Brisbane (Australia) Endpoint" ,
} , {
Value : "au-vic.s3.rackcorp.com" ,
Help : "Melbourne (Australia) Endpoint" ,
} , {
Value : "au-wa.s3.rackcorp.com" ,
Help : "Perth (Australia) Endpoint" ,
} , {
Value : "ph.s3.rackcorp.com" ,
Help : "Manila (Philippines) Endpoint" ,
} , {
Value : "th.s3.rackcorp.com" ,
Help : "Bangkok (Thailand) Endpoint" ,
} , {
Value : "hk.s3.rackcorp.com" ,
Help : "HK (Hong Kong) Endpoint" ,
} , {
Value : "mn.s3.rackcorp.com" ,
Help : "Ulaanbaatar (Mongolia) Endpoint" ,
} , {
Value : "kg.s3.rackcorp.com" ,
Help : "Bishkek (Kyrgyzstan) Endpoint" ,
} , {
Value : "id.s3.rackcorp.com" ,
Help : "Jakarta (Indonesia) Endpoint" ,
} , {
Value : "jp.s3.rackcorp.com" ,
Help : "Tokyo (Japan) Endpoint" ,
} , {
Value : "sg.s3.rackcorp.com" ,
Help : "SG (Singapore) Endpoint" ,
} , {
Value : "de.s3.rackcorp.com" ,
Help : "Frankfurt (Germany) Endpoint" ,
} , {
Value : "us.s3.rackcorp.com" ,
Help : "USA (AnyCast) Endpoint" ,
} , {
Value : "us-east-1.s3.rackcorp.com" ,
Help : "New York (USA) Endpoint" ,
} , {
Value : "us-west-1.s3.rackcorp.com" ,
Help : "Freemont (USA) Endpoint" ,
} , {
Value : "nz.s3.rackcorp.com" ,
Help : "Auckland (New Zealand) Endpoint" ,
} } ,
2022-10-13 16:49:22 +02:00
} , {
// Qiniu endpoints: https://developer.qiniu.com/kodo/4088/s3-access-domainname
Name : "endpoint" ,
Help : "Endpoint for Qiniu Object Storage." ,
Provider : "Qiniu" ,
Examples : [ ] fs . OptionExample { {
Value : "s3-cn-east-1.qiniucs.com" ,
Help : "East China Endpoint 1" ,
} , {
Value : "s3-cn-east-2.qiniucs.com" ,
Help : "East China Endpoint 2" ,
} , {
Value : "s3-cn-north-1.qiniucs.com" ,
Help : "North China Endpoint 1" ,
} , {
Value : "s3-cn-south-1.qiniucs.com" ,
Help : "South China Endpoint 1" ,
} , {
Value : "s3-us-north-1.qiniucs.com" ,
Help : "North America Endpoint 1" ,
} , {
Value : "s3-ap-southeast-1.qiniucs.com" ,
Help : "Southeast Asia Endpoint 1" ,
} , {
Value : "s3-ap-northeast-1.qiniucs.com" ,
Help : "Northeast Asia Endpoint 1" ,
} } ,
2018-04-13 17:08:00 +02:00
} , {
Name : "endpoint" ,
2021-08-16 11:30:01 +02:00
Help : "Endpoint for S3 API.\n\nRequired when using an S3 clone." ,
2024-06-06 12:25:45 +02:00
Provider : "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,MagaluCloud,Scaleway,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox" ,
2018-04-13 17:08:00 +02:00
Examples : [ ] fs . OptionExample { {
2019-02-13 22:10:43 +01:00
Value : "objects-us-east-1.dream.io" ,
2018-04-13 17:08:00 +02:00
Help : "Dream Objects endpoint" ,
Provider : "Dreamhost" ,
2022-12-14 15:00:04 +01:00
} , {
Value : "syd1.digitaloceanspaces.com" ,
Help : "DigitalOcean Spaces Sydney 1" ,
Provider : "DigitalOcean" ,
} , {
Value : "sfo3.digitaloceanspaces.com" ,
Help : "DigitalOcean Spaces San Francisco 3" ,
Provider : "DigitalOcean" ,
} , {
Value : "fra1.digitaloceanspaces.com" ,
Help : "DigitalOcean Spaces Frankfurt 1" ,
Provider : "DigitalOcean" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "nyc3.digitaloceanspaces.com" ,
2022-12-14 15:00:04 +01:00
Help : "DigitalOcean Spaces New York 3" ,
2018-04-13 17:08:00 +02:00
Provider : "DigitalOcean" ,
} , {
Value : "ams3.digitaloceanspaces.com" ,
2022-12-14 15:00:04 +01:00
Help : "DigitalOcean Spaces Amsterdam 3" ,
2018-04-13 17:08:00 +02:00
Provider : "DigitalOcean" ,
} , {
Value : "sgp1.digitaloceanspaces.com" ,
2022-12-14 15:00:04 +01:00
Help : "DigitalOcean Spaces Singapore 1" ,
2018-04-13 17:08:00 +02:00
Provider : "DigitalOcean" ,
} , {
2021-06-08 10:59:57 +02:00
Value : "localhost:8333" ,
Help : "SeaweedFS S3 localhost" ,
Provider : "SeaweedFS" ,
2022-03-04 13:35:46 +01:00
} , {
Value : "s3.us-east-1.lyvecloud.seagate.com" ,
2022-03-14 16:37:34 +01:00
Help : "Seagate Lyve Cloud US East 1 (Virginia)" ,
Provider : "LyveCloud" ,
} , {
Value : "s3.us-west-1.lyvecloud.seagate.com" ,
Help : "Seagate Lyve Cloud US West 1 (California)" ,
Provider : "LyveCloud" ,
} , {
Value : "s3.ap-southeast-1.lyvecloud.seagate.com" ,
Help : "Seagate Lyve Cloud AP Southeast 1 (Singapore)" ,
Provider : "LyveCloud" ,
2021-06-08 10:59:57 +02:00
} , {
2018-04-13 17:08:00 +02:00
Value : "s3.wasabisys.com" ,
2022-11-11 06:19:27 +01:00
Help : "Wasabi US East 1 (N. Virginia)" ,
Provider : "Wasabi" ,
} , {
Value : "s3.us-east-2.wasabisys.com" ,
Help : "Wasabi US East 2 (N. Virginia)" ,
Provider : "Wasabi" ,
} , {
Value : "s3.us-central-1.wasabisys.com" ,
Help : "Wasabi US Central 1 (Texas)" ,
2018-11-17 17:24:00 +01:00
Provider : "Wasabi" ,
} , {
Value : "s3.us-west-1.wasabisys.com" ,
2022-11-11 06:19:27 +01:00
Help : "Wasabi US West 1 (Oregon)" ,
Provider : "Wasabi" ,
} , {
Value : "s3.ca-central-1.wasabisys.com" ,
Help : "Wasabi CA Central 1 (Toronto)" ,
2018-04-13 17:08:00 +02:00
Provider : "Wasabi" ,
2019-05-15 10:22:06 +02:00
} , {
Value : "s3.eu-central-1.wasabisys.com" ,
2022-11-11 06:19:27 +01:00
Help : "Wasabi EU Central 1 (Amsterdam)" ,
Provider : "Wasabi" ,
} , {
Value : "s3.eu-central-2.wasabisys.com" ,
Help : "Wasabi EU Central 2 (Frankfurt)" ,
Provider : "Wasabi" ,
} , {
Value : "s3.eu-west-1.wasabisys.com" ,
Help : "Wasabi EU West 1 (London)" ,
Provider : "Wasabi" ,
} , {
Value : "s3.eu-west-2.wasabisys.com" ,
Help : "Wasabi EU West 2 (Paris)" ,
2019-05-15 10:22:06 +02:00
Provider : "Wasabi" ,
2021-07-30 18:47:33 +02:00
} , {
Value : "s3.ap-northeast-1.wasabisys.com" ,
2021-11-21 22:04:53 +01:00
Help : "Wasabi AP Northeast 1 (Tokyo) endpoint" ,
Provider : "Wasabi" ,
} , {
Value : "s3.ap-northeast-2.wasabisys.com" ,
Help : "Wasabi AP Northeast 2 (Osaka) endpoint" ,
2021-07-30 18:47:33 +02:00
Provider : "Wasabi" ,
2022-11-11 06:19:27 +01:00
} , {
Value : "s3.ap-southeast-1.wasabisys.com" ,
Help : "Wasabi AP Southeast 1 (Singapore)" ,
Provider : "Wasabi" ,
} , {
Value : "s3.ap-southeast-2.wasabisys.com" ,
Help : "Wasabi AP Southeast 2 (Sydney)" ,
Provider : "Wasabi" ,
2022-12-06 13:25:23 +01:00
} , {
Value : "storage.iran.liara.space" ,
Help : "Liara Iran endpoint" ,
Provider : "Liara" ,
2022-04-28 11:42:30 +02:00
} , {
2023-06-17 13:47:23 +02:00
Value : "s3.ir-thr-at1.arvanstorage.ir" ,
Help : "ArvanCloud Tehran Iran (Simin) endpoint" ,
Provider : "ArvanCloud" ,
} , {
Value : "s3.ir-tbz-sh1.arvanstorage.ir" ,
Help : "ArvanCloud Tabriz Iran (Shahriar) endpoint" ,
2022-04-28 11:42:30 +02:00
Provider : "ArvanCloud" ,
2024-06-06 12:25:45 +02:00
} , {
Value : "br-se1.magaluobjects.com" ,
Help : "Magalu BR Southeast 1 endpoint" ,
Provider : "Magalu" ,
} , {
2024-06-15 18:40:28 +02:00
Value : "br-ne1.magaluobjects.com" ,
Help : "Magalu BR Northeast 1 endpoint" ,
2024-06-06 12:25:45 +02:00
Provider : "Magalu" ,
2018-04-13 17:08:00 +02:00
} } ,
} , {
Name : "location_constraint" ,
2021-08-16 11:30:01 +02:00
Help : "Location constraint - must be set to match the Region.\n\nUsed when creating buckets only." ,
2018-04-13 17:08:00 +02:00
Provider : "AWS" ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
2021-08-16 11:30:01 +02:00
Help : "Empty for US Region, Northern Virginia, or Pacific Northwest" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "us-east-2" ,
2021-08-16 11:30:01 +02:00
Help : "US East (Ohio) Region" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "us-west-1" ,
2021-08-16 11:30:01 +02:00
Help : "US West (Northern California) Region" ,
2020-10-03 10:44:25 +02:00
} , {
Value : "us-west-2" ,
2021-08-16 11:30:01 +02:00
Help : "US West (Oregon) Region" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ca-central-1" ,
2021-08-16 11:30:01 +02:00
Help : "Canada (Central) Region" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "eu-west-1" ,
2021-08-16 11:30:01 +02:00
Help : "EU (Ireland) Region" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "eu-west-2" ,
2021-08-16 11:30:01 +02:00
Help : "EU (London) Region" ,
2020-10-03 10:44:25 +02:00
} , {
Value : "eu-west-3" ,
2021-08-16 11:30:01 +02:00
Help : "EU (Paris) Region" ,
2019-02-02 20:44:26 +01:00
} , {
Value : "eu-north-1" ,
2021-08-16 11:30:01 +02:00
Help : "EU (Stockholm) Region" ,
2020-10-03 10:44:25 +02:00
} , {
Value : "eu-south-1" ,
2021-08-16 11:30:01 +02:00
Help : "EU (Milan) Region" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "EU" ,
2021-08-16 11:30:01 +02:00
Help : "EU Region" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ap-southeast-1" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Singapore) Region" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ap-southeast-2" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Sydney) Region" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ap-northeast-1" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Tokyo) Region" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ap-northeast-2" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Seoul) Region" ,
2020-10-03 10:44:25 +02:00
} , {
Value : "ap-northeast-3" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Osaka-Local) Region" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ap-south-1" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Mumbai) Region" ,
2020-01-02 12:10:48 +01:00
} , {
Value : "ap-east-1" ,
2021-08-16 11:30:01 +02:00
Help : "Asia Pacific (Hong Kong) Region" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "sa-east-1" ,
2021-08-16 11:30:01 +02:00
Help : "South America (Sao Paulo) Region" ,
2024-04-01 19:17:16 +02:00
} , {
Value : "il-central-1" ,
Help : "Israel (Tel Aviv) Region" ,
2020-10-03 10:44:25 +02:00
} , {
Value : "me-south-1" ,
2021-08-16 11:30:01 +02:00
Help : "Middle East (Bahrain) Region" ,
2020-10-03 10:44:25 +02:00
} , {
Value : "af-south-1" ,
2021-08-16 11:30:01 +02:00
Help : "Africa (Cape Town) Region" ,
2020-10-03 10:44:25 +02:00
} , {
Value : "cn-north-1" ,
Help : "China (Beijing) Region" ,
} , {
Value : "cn-northwest-1" ,
2021-08-16 11:30:01 +02:00
Help : "China (Ningxia) Region" ,
2020-10-03 10:44:25 +02:00
} , {
Value : "us-gov-east-1" ,
2021-08-16 11:30:01 +02:00
Help : "AWS GovCloud (US-East) Region" ,
2020-10-03 10:44:25 +02:00
} , {
Value : "us-gov-west-1" ,
2021-08-16 11:30:01 +02:00
Help : "AWS GovCloud (US) Region" ,
2018-04-13 17:08:00 +02:00
} } ,
2022-03-24 12:57:00 +01:00
} , {
Name : "location_constraint" ,
Help : "Location constraint - must match endpoint.\n\nUsed when creating buckets only." ,
Provider : "ChinaMobile" ,
Examples : [ ] fs . OptionExample { {
Value : "wuxi1" ,
Help : "East China (Suzhou)" ,
} , {
Value : "jinan1" ,
Help : "East China (Jinan)" ,
} , {
Value : "ningbo1" ,
Help : "East China (Hangzhou)" ,
} , {
Value : "shanghai1" ,
Help : "East China (Shanghai-1)" ,
} , {
Value : "zhengzhou1" ,
Help : "Central China (Zhengzhou)" ,
} , {
Value : "hunan1" ,
Help : "Central China (Changsha-1)" ,
} , {
Value : "zhuzhou1" ,
Help : "Central China (Changsha-2)" ,
} , {
Value : "guangzhou1" ,
Help : "South China (Guangzhou-2)" ,
} , {
Value : "dongguan1" ,
Help : "South China (Guangzhou-3)" ,
} , {
Value : "beijing1" ,
Help : "North China (Beijing-1)" ,
} , {
Value : "beijing2" ,
Help : "North China (Beijing-2)" ,
} , {
Value : "beijing4" ,
Help : "North China (Beijing-3)" ,
} , {
Value : "huhehaote1" ,
Help : "North China (Huhehaote)" ,
} , {
Value : "chengdu1" ,
Help : "Southwest China (Chengdu)" ,
} , {
Value : "chongqing1" ,
Help : "Southwest China (Chongqing)" ,
} , {
Value : "guiyang1" ,
Help : "Southwest China (Guiyang)" ,
} , {
Value : "xian1" ,
Help : "Nouthwest China (Xian)" ,
} , {
Value : "yunnan" ,
Help : "Yunnan China (Kunming)" ,
} , {
Value : "yunnan2" ,
Help : "Yunnan China (Kunming-2)" ,
} , {
Value : "tianjin1" ,
Help : "Tianjin China (Tianjin)" ,
} , {
Value : "jilin1" ,
Help : "Jilin China (Changchun)" ,
} , {
Value : "hubei1" ,
Help : "Hubei China (Xiangyan)" ,
} , {
Value : "jiangxi1" ,
Help : "Jiangxi China (Nanchang)" ,
} , {
Value : "gansu1" ,
Help : "Gansu China (Lanzhou)" ,
} , {
Value : "shanxi1" ,
Help : "Shanxi China (Taiyuan)" ,
} , {
Value : "liaoning1" ,
Help : "Liaoning China (Shenyang)" ,
} , {
Value : "hebei1" ,
Help : "Hebei China (Shijiazhuang)" ,
} , {
Value : "fujian1" ,
Help : "Fujian China (Xiamen)" ,
} , {
Value : "guangxi1" ,
Help : "Guangxi China (Nanning)" ,
} , {
Value : "anhui1" ,
Help : "Anhui China (Huainan)" ,
} } ,
2022-04-28 11:42:30 +02:00
} , {
Name : "location_constraint" ,
Help : "Location constraint - must match endpoint.\n\nUsed when creating buckets only." ,
Provider : "ArvanCloud" ,
Examples : [ ] fs . OptionExample { {
Value : "ir-thr-at1" ,
2023-06-17 13:47:23 +02:00
Help : "Tehran Iran (Simin)" ,
2022-04-28 11:42:30 +02:00
} , {
Value : "ir-tbz-sh1" ,
Help : "Tabriz Iran (Shahriar)" ,
} } ,
2018-04-13 17:08:00 +02:00
} , {
Name : "location_constraint" ,
2021-08-16 11:30:01 +02:00
Help : "Location constraint - must match endpoint when using IBM Cloud Public.\n\nFor on-prem COS, do not make a selection from this list, hit enter." ,
2018-04-13 17:08:00 +02:00
Provider : "IBMCOS" ,
Examples : [ ] fs . OptionExample { {
Value : "us-standard" ,
Help : "US Cross Region Standard" ,
} , {
Value : "us-vault" ,
Help : "US Cross Region Vault" ,
} , {
Value : "us-cold" ,
Help : "US Cross Region Cold" ,
} , {
Value : "us-flex" ,
Help : "US Cross Region Flex" ,
} , {
Value : "us-east-standard" ,
Help : "US East Region Standard" ,
} , {
Value : "us-east-vault" ,
Help : "US East Region Vault" ,
} , {
Value : "us-east-cold" ,
Help : "US East Region Cold" ,
} , {
Value : "us-east-flex" ,
Help : "US East Region Flex" ,
} , {
Value : "us-south-standard" ,
2019-02-07 18:41:17 +01:00
Help : "US South Region Standard" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "us-south-vault" ,
Help : "US South Region Vault" ,
} , {
Value : "us-south-cold" ,
Help : "US South Region Cold" ,
} , {
Value : "us-south-flex" ,
Help : "US South Region Flex" ,
} , {
Value : "eu-standard" ,
Help : "EU Cross Region Standard" ,
} , {
Value : "eu-vault" ,
Help : "EU Cross Region Vault" ,
} , {
Value : "eu-cold" ,
Help : "EU Cross Region Cold" ,
} , {
Value : "eu-flex" ,
Help : "EU Cross Region Flex" ,
} , {
Value : "eu-gb-standard" ,
2019-02-07 18:41:17 +01:00
Help : "Great Britain Standard" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "eu-gb-vault" ,
2019-02-07 18:41:17 +01:00
Help : "Great Britain Vault" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "eu-gb-cold" ,
2019-02-07 18:41:17 +01:00
Help : "Great Britain Cold" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "eu-gb-flex" ,
2019-02-07 18:41:17 +01:00
Help : "Great Britain Flex" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "ap-standard" ,
Help : "APAC Standard" ,
} , {
Value : "ap-vault" ,
Help : "APAC Vault" ,
} , {
Value : "ap-cold" ,
Help : "APAC Cold" ,
} , {
Value : "ap-flex" ,
Help : "APAC Flex" ,
} , {
Value : "mel01-standard" ,
Help : "Melbourne Standard" ,
} , {
Value : "mel01-vault" ,
Help : "Melbourne Vault" ,
} , {
Value : "mel01-cold" ,
Help : "Melbourne Cold" ,
} , {
Value : "mel01-flex" ,
Help : "Melbourne Flex" ,
} , {
Value : "tor01-standard" ,
Help : "Toronto Standard" ,
} , {
Value : "tor01-vault" ,
Help : "Toronto Vault" ,
} , {
Value : "tor01-cold" ,
Help : "Toronto Cold" ,
} , {
Value : "tor01-flex" ,
Help : "Toronto Flex" ,
} } ,
2021-11-09 12:46:58 +01:00
} , {
Name : "location_constraint" ,
Help : "Location constraint - the location where your bucket will be located and your data stored.\n" ,
Provider : "RackCorp" ,
Examples : [ ] fs . OptionExample { {
Value : "global" ,
Help : "Global CDN Region" ,
} , {
Value : "au" ,
Help : "Australia (All locations)" ,
} , {
Value : "au-nsw" ,
Help : "NSW (Australia) Region" ,
} , {
Value : "au-qld" ,
Help : "QLD (Australia) Region" ,
} , {
Value : "au-vic" ,
Help : "VIC (Australia) Region" ,
} , {
Value : "au-wa" ,
Help : "Perth (Australia) Region" ,
} , {
Value : "ph" ,
Help : "Manila (Philippines) Region" ,
} , {
Value : "th" ,
Help : "Bangkok (Thailand) Region" ,
} , {
Value : "hk" ,
Help : "HK (Hong Kong) Region" ,
} , {
Value : "mn" ,
Help : "Ulaanbaatar (Mongolia) Region" ,
} , {
Value : "kg" ,
Help : "Bishkek (Kyrgyzstan) Region" ,
} , {
Value : "id" ,
Help : "Jakarta (Indonesia) Region" ,
} , {
Value : "jp" ,
Help : "Tokyo (Japan) Region" ,
} , {
Value : "sg" ,
Help : "SG (Singapore) Region" ,
} , {
Value : "de" ,
Help : "Frankfurt (Germany) Region" ,
} , {
Value : "us" ,
Help : "USA (AnyCast) Region" ,
} , {
Value : "us-east-1" ,
Help : "New York (USA) Region" ,
} , {
Value : "us-west-1" ,
Help : "Freemont (USA) Region" ,
} , {
Value : "nz" ,
Help : "Auckland (New Zealand) Region" ,
} } ,
2022-10-13 16:49:22 +02:00
} , {
Name : "location_constraint" ,
Help : "Location constraint - must be set to match the Region.\n\nUsed when creating buckets only." ,
Provider : "Qiniu" ,
Examples : [ ] fs . OptionExample { {
Value : "cn-east-1" ,
Help : "East China Region 1" ,
} , {
Value : "cn-east-2" ,
Help : "East China Region 2" ,
} , {
Value : "cn-north-1" ,
Help : "North China Region 1" ,
} , {
Value : "cn-south-1" ,
Help : "South China Region 1" ,
} , {
Value : "us-north-1" ,
Help : "North America Region 1" ,
} , {
Value : "ap-southeast-1" ,
Help : "Southeast Asia Region 1" ,
} , {
Value : "ap-northeast-1" ,
Help : "Northeast Asia Region 1" ,
} } ,
2018-04-13 17:08:00 +02:00
} , {
Name : "location_constraint" ,
2021-08-16 11:30:01 +02:00
Help : "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only." ,
2024-06-06 12:25:45 +02:00
Provider : "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Leviia,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS,Petabox" ,
2018-04-13 17:08:00 +02:00
} , {
Name : "acl" ,
2018-10-25 23:19:36 +02:00
Help : ` Canned ACL used when creating buckets and storing or copying objects .
2019-01-16 18:23:37 +01:00
This ACL is used for creating objects and if bucket_acl isn ' t set , for creating buckets too .
2018-10-25 23:19:36 +02:00
For more info visit https : //docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
2020-10-13 23:43:40 +02:00
Note that this ACL is applied when server - side copying objects as S3
2022-09-14 17:24:57 +02:00
doesn ' t copy the ACL from the source but rather writes a fresh one .
If the acl is an empty string then no X - Amz - Acl : header is added and
the default ( private ) will be used .
` ,
2023-07-06 11:54:07 +02:00
Provider : "!Storj,Synology,Cloudflare" ,
2018-04-13 17:08:00 +02:00
Examples : [ ] fs . OptionExample { {
2020-09-08 17:34:25 +02:00
Value : "default" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets Full_CONTROL.\nNo one else has access rights (default)." ,
2020-09-08 17:34:25 +02:00
Provider : "TencentCOS" ,
} , {
2018-04-13 17:08:00 +02:00
Value : "private" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nNo one else has access rights (default)." ,
2020-09-08 17:34:25 +02:00
Provider : "!IBMCOS,TencentCOS" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "public-read" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nThe AllUsers group gets READ access." ,
2018-04-13 17:08:00 +02:00
Provider : "!IBMCOS" ,
} , {
Value : "public-read-write" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nThe AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended." ,
2018-04-13 17:08:00 +02:00
Provider : "!IBMCOS" ,
} , {
Value : "authenticated-read" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nThe AuthenticatedUsers group gets READ access." ,
2018-04-13 17:08:00 +02:00
Provider : "!IBMCOS" ,
} , {
Value : "bucket-owner-read" ,
2021-08-16 11:30:01 +02:00
Help : "Object owner gets FULL_CONTROL.\nBucket owner gets READ access.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it." ,
2022-03-24 12:57:00 +01:00
Provider : "!IBMCOS,ChinaMobile" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "bucket-owner-full-control" ,
Help : "Both the object owner and the bucket owner get FULL_CONTROL over the object.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it." ,
2022-03-24 12:57:00 +01:00
Provider : "!IBMCOS,ChinaMobile" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "private" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nNo one else has access rights (default).\nThis acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise COS." ,
2018-04-12 18:05:53 +02:00
Provider : "IBMCOS" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "public-read" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nThe AllUsers group gets READ access.\nThis acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise IBM COS." ,
2018-04-13 17:08:00 +02:00
Provider : "IBMCOS" ,
} , {
Value : "public-read-write" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nThe AllUsers group gets READ and WRITE access.\nThis acl is available on IBM Cloud (Infra), On-Premise IBM COS." ,
2018-04-12 18:05:53 +02:00
Provider : "IBMCOS" ,
2018-04-13 17:08:00 +02:00
} , {
Value : "authenticated-read" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nThe AuthenticatedUsers group gets READ access.\nNot supported on Buckets.\nThis acl is available on IBM Cloud (Infra) and On-Premise IBM COS." ,
2018-04-13 17:08:00 +02:00
Provider : "IBMCOS" ,
} } ,
2019-01-16 18:23:37 +01:00
} , {
Name : "bucket_acl" ,
Help : ` Canned ACL used when creating buckets .
For more info visit https : //docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
Note that this ACL is applied when only when creating buckets . If it
2022-09-14 17:24:57 +02:00
isn ' t set then "acl" is used instead .
If the "acl" and "bucket_acl" are empty strings then no X - Amz - Acl :
header is added and the default ( private ) will be used .
` ,
2019-01-16 18:23:37 +01:00
Advanced : true ,
Examples : [ ] fs . OptionExample { {
Value : "private" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nNo one else has access rights (default)." ,
2019-01-16 18:23:37 +01:00
} , {
Value : "public-read" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nThe AllUsers group gets READ access." ,
2019-01-16 18:23:37 +01:00
} , {
Value : "public-read-write" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nThe AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended." ,
2019-01-16 18:23:37 +01:00
} , {
Value : "authenticated-read" ,
2021-08-16 11:30:01 +02:00
Help : "Owner gets FULL_CONTROL.\nThe AuthenticatedUsers group gets READ access." ,
2019-01-16 18:23:37 +01:00
} } ,
2020-12-03 03:30:06 +01:00
} , {
Name : "requester_pays" ,
Help : "Enables requester pays option when interacting with S3 bucket." ,
Provider : "AWS" ,
Default : false ,
Advanced : true ,
2018-04-13 17:08:00 +02:00
} , {
Name : "server_side_encryption" ,
Help : "The server-side encryption algorithm used when storing this object in S3." ,
2022-05-02 17:04:01 +02:00
Provider : "AWS,Ceph,ChinaMobile,Minio" ,
2018-04-13 17:08:00 +02:00
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "None" ,
} , {
Value : "AES256" ,
Help : "AES256" ,
2018-08-30 18:08:27 +02:00
} , {
2022-03-24 12:57:00 +01:00
Value : "aws:kms" ,
Help : "aws:kms" ,
Provider : "!ChinaMobile" ,
2018-08-30 18:08:27 +02:00
} } ,
2020-03-30 12:26:52 +02:00
} , {
Name : "sse_customer_algorithm" ,
Help : "If using SSE-C, the server-side encryption algorithm used when storing this object in S3." ,
2022-05-02 17:04:01 +02:00
Provider : "AWS,Ceph,ChinaMobile,Minio" ,
2020-03-30 12:26:52 +02:00
Advanced : true ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "None" ,
} , {
Value : "AES256" ,
Help : "AES256" ,
} } ,
2018-08-30 18:08:27 +02:00
} , {
Name : "sse_kms_key_id" ,
Help : "If using KMS ID you must provide the ARN of Key." ,
2022-05-02 17:04:01 +02:00
Provider : "AWS,Ceph,Minio" ,
2018-08-30 18:08:27 +02:00
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "None" ,
} , {
Value : "arn:aws:kms:us-east-1:*" ,
Help : "arn:aws:kms:*" ,
2018-04-13 17:08:00 +02:00
} } ,
2023-07-06 18:55:53 +02:00
Sensitive : true ,
2020-03-30 12:26:52 +02:00
} , {
2022-09-17 18:28:44 +02:00
Name : "sse_customer_key" ,
Help : ` To use SSE - C you may provide the secret encryption key used to encrypt / decrypt your data .
Alternatively you can provide -- sse - customer - key - base64 . ` ,
Provider : "AWS,Ceph,ChinaMobile,Minio" ,
Advanced : true ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "None" ,
} } ,
2023-07-06 18:55:53 +02:00
Sensitive : true ,
2022-09-17 18:28:44 +02:00
} , {
Name : "sse_customer_key_base64" ,
Help : ` If using SSE - C you must provide the secret encryption key encoded in base64 format to encrypt / decrypt your data .
Alternatively you can provide -- sse - customer - key . ` ,
2022-05-02 17:04:01 +02:00
Provider : "AWS,Ceph,ChinaMobile,Minio" ,
2020-03-30 12:26:52 +02:00
Advanced : true ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "None" ,
} } ,
2023-07-06 18:55:53 +02:00
Sensitive : true ,
2020-03-30 12:26:52 +02:00
} , {
2020-11-20 12:15:48 +01:00
Name : "sse_customer_key_md5" ,
Help : ` If using SSE - C you may provide the secret encryption key MD5 checksum ( optional ) .
If you leave it blank , this is calculated automatically from the sse_customer_key provided .
` ,
2022-05-02 17:04:01 +02:00
Provider : "AWS,Ceph,ChinaMobile,Minio" ,
2020-03-30 12:26:52 +02:00
Advanced : true ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "None" ,
} } ,
2023-07-06 18:55:53 +02:00
Sensitive : true ,
2018-04-13 17:08:00 +02:00
} , {
Name : "storage_class" ,
2018-10-01 19:36:15 +02:00
Help : "The storage class to use when storing new objects in S3." ,
2018-04-13 17:08:00 +02:00
Provider : "AWS" ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "Default" ,
} , {
Value : "STANDARD" ,
Help : "Standard storage class" ,
} , {
Value : "REDUCED_REDUNDANCY" ,
Help : "Reduced redundancy storage class" ,
} , {
Value : "STANDARD_IA" ,
Help : "Standard Infrequent Access storage class" ,
} , {
Value : "ONEZONE_IA" ,
Help : "One Zone Infrequent Access storage class" ,
2018-12-06 22:53:05 +01:00
} , {
Value : "GLACIER" ,
Help : "Glacier storage class" ,
2019-04-05 05:14:05 +02:00
} , {
Value : "DEEP_ARCHIVE" ,
Help : "Glacier Deep Archive storage class" ,
2019-04-04 19:04:28 +02:00
} , {
Value : "INTELLIGENT_TIERING" ,
Help : "Intelligent-Tiering storage class" ,
2021-12-02 06:55:57 +01:00
} , {
Value : "GLACIER_IR" ,
Help : "Glacier Instant Retrieval storage class" ,
2018-04-13 17:08:00 +02:00
} } ,
2019-01-12 17:46:45 +01:00
} , {
2019-01-12 21:41:47 +01:00
// Mapping from here: https://www.alibabacloud.com/help/doc-detail/64919.htm
2019-01-12 17:46:45 +01:00
Name : "storage_class" ,
Help : "The storage class to use when storing new objects in OSS." ,
Provider : "Alibaba" ,
Examples : [ ] fs . OptionExample { {
2019-01-12 21:41:47 +01:00
Value : "" ,
Help : "Default" ,
} , {
Value : "STANDARD" ,
2019-01-12 17:46:45 +01:00
Help : "Standard storage class" ,
} , {
2019-01-12 21:41:47 +01:00
Value : "GLACIER" ,
2021-08-16 11:30:01 +02:00
Help : "Archive storage mode" ,
2019-01-12 17:46:45 +01:00
} , {
2019-01-12 21:41:47 +01:00
Value : "STANDARD_IA" ,
2021-08-16 11:30:01 +02:00
Help : "Infrequent access storage mode" ,
2019-01-12 17:46:45 +01:00
} } ,
2022-03-24 12:57:00 +01:00
} , {
// Mapping from here: https://ecloud.10086.cn/op-help-center/doc/article/24495
Name : "storage_class" ,
Help : "The storage class to use when storing new objects in ChinaMobile." ,
Provider : "ChinaMobile" ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "Default" ,
} , {
Value : "STANDARD" ,
Help : "Standard storage class" ,
} , {
Value : "GLACIER" ,
Help : "Archive storage mode" ,
} , {
Value : "STANDARD_IA" ,
Help : "Infrequent access storage mode" ,
} } ,
2022-12-06 13:25:23 +01:00
} , {
// Mapping from here: https://liara.ir/landing/object-storage
Name : "storage_class" ,
Help : "The storage class to use when storing new objects in Liara" ,
Provider : "Liara" ,
Examples : [ ] fs . OptionExample { {
Value : "STANDARD" ,
Help : "Standard storage class" ,
} } ,
2022-04-28 11:42:30 +02:00
} , {
2023-06-17 13:47:23 +02:00
// Mapping from here: https://www.arvancloud.ir/en/products/cloud-storage
2022-04-28 11:42:30 +02:00
Name : "storage_class" ,
Help : "The storage class to use when storing new objects in ArvanCloud." ,
Provider : "ArvanCloud" ,
Examples : [ ] fs . OptionExample { {
Value : "STANDARD" ,
Help : "Standard storage class" ,
} } ,
2024-06-06 12:25:45 +02:00
} , {
// Mapping from here: #todo
Name : "storage_class" ,
Help : "The storage class to use when storing new objects in Magalu." ,
Provider : "Magalu" ,
Examples : [ ] fs . OptionExample { {
Value : "STANDARD" ,
Help : "Standard storage class" ,
} } ,
2020-09-08 17:34:25 +02:00
} , {
// Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925
Name : "storage_class" ,
Help : "The storage class to use when storing new objects in Tencent COS." ,
Provider : "TencentCOS" ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
Help : "Default" ,
} , {
Value : "STANDARD" ,
Help : "Standard storage class" ,
} , {
Value : "ARCHIVE" ,
2021-08-16 11:30:01 +02:00
Help : "Archive storage mode" ,
2020-09-08 17:34:25 +02:00
} , {
Value : "STANDARD_IA" ,
2021-08-16 11:30:01 +02:00
Help : "Infrequent access storage mode" ,
2020-09-08 17:34:25 +02:00
} } ,
2020-06-12 17:04:16 +02:00
} , {
2023-04-18 23:21:35 +02:00
// Mapping from here: https://www.scaleway.com/en/docs/storage/object/quickstart/
2020-06-12 17:04:16 +02:00
Name : "storage_class" ,
Help : "The storage class to use when storing new objects in S3." ,
Provider : "Scaleway" ,
Examples : [ ] fs . OptionExample { {
Value : "" ,
2021-08-16 11:30:01 +02:00
Help : "Default." ,
2020-06-12 17:04:16 +02:00
} , {
Value : "STANDARD" ,
2023-04-18 23:21:35 +02:00
Help : "The Standard class for any upload.\nSuitable for on-demand content like streaming or CDN.\nAvailable in all regions." ,
2020-06-12 17:04:16 +02:00
} , {
Value : "GLACIER" ,
2023-04-18 23:21:35 +02:00
Help : "Archived storage.\nPrices are lower, but it needs to be restored first to be accessed.\nAvailable in FR-PAR and NL-AMS regions." ,
} , {
Value : "ONEZONE_IA" ,
Help : "One Zone - Infrequent Access.\nA good choice for storing secondary backup copies or easily re-creatable data.\nAvailable in the FR-PAR region only." ,
2020-06-12 17:04:16 +02:00
} } ,
2022-10-13 16:49:22 +02:00
} , {
// Mapping from here: https://developer.qiniu.com/kodo/5906/storage-type
Name : "storage_class" ,
Help : "The storage class to use when storing new objects in Qiniu." ,
Provider : "Qiniu" ,
Examples : [ ] fs . OptionExample { {
Value : "STANDARD" ,
Help : "Standard storage class" ,
} , {
Value : "LINE" ,
Help : "Infrequent access storage mode" ,
} , {
Value : "GLACIER" ,
Help : "Archive storage mode" ,
} , {
Value : "DEEP_ARCHIVE" ,
Help : "Deep archive storage mode" ,
} } ,
2018-11-26 22:09:23 +01:00
} , {
Name : "upload_cutoff" ,
2021-08-16 11:30:01 +02:00
Help : ` Cutoff for switching to chunked upload .
2018-11-26 22:09:23 +01:00
Any files larger than this will be uploaded in chunks of chunk_size .
2021-03-02 20:11:57 +01:00
The minimum is 0 and the maximum is 5 GiB . ` ,
2018-11-26 22:09:23 +01:00
Default : defaultUploadCutoff ,
Advanced : true ,
2018-05-14 19:06:57 +02:00
} , {
2018-10-01 19:36:15 +02:00
Name : "chunk_size" ,
Help : ` Chunk size to use for uploading .
2019-11-06 11:41:03 +01:00
When uploading files larger than upload_cutoff or files with unknown
2020-10-13 23:49:58 +02:00
size ( e . g . from "rclone rcat" or uploaded with "rclone mount" or google
2019-11-06 11:41:03 +01:00
photos or google docs ) they will be uploaded as multipart uploads
using this chunk size .
2018-10-01 19:36:15 +02:00
Note that "--s3-upload-concurrency" chunks of this size are buffered
in memory per transfer .
2020-10-13 23:50:53 +02:00
If you are transferring large files over high - speed links and you have
2019-11-06 11:41:03 +01:00
enough memory , then increasing this will speed up the transfers .
Rclone will automatically increase the chunk size when uploading a
large file of known size to stay below the 10 , 000 chunks limit .
Files of unknown size are uploaded with the configured
2021-03-02 20:11:57 +01:00
chunk_size . Since the default chunk size is 5 MiB and there can be at
2019-11-06 11:41:03 +01:00
most 10 , 000 chunks , this means that by default the maximum size of
2021-03-02 20:11:57 +01:00
a file you can stream upload is 48 GiB . If you wish to stream upload
2022-06-16 22:29:36 +02:00
larger files then you will need to increase chunk_size .
Increasing the chunk size decreases the accuracy of the progress
statistics displayed with "-P" flag . Rclone treats chunk as sent when
it ' s buffered by the AWS SDK , when in fact it may still be uploading .
A bigger chunk size means a bigger AWS SDK buffer and progress
reporting more deviating from the truth .
` ,
2018-09-07 13:02:27 +02:00
Default : minChunkSize ,
2018-05-14 19:06:57 +02:00
Advanced : true ,
2020-06-08 19:22:34 +02:00
} , {
Name : "max_upload_parts" ,
Help : ` Maximum number of parts in a multipart upload .
This option defines the maximum number of multipart chunks to use
when doing a multipart upload .
This can be useful if a service does not support the AWS S3
specification of 10 , 000 chunks .
Rclone will automatically increase the chunk size when uploading a
large file of a known size to stay below this number of chunks limit .
` ,
Default : maxUploadParts ,
Advanced : true ,
2019-12-02 18:14:57 +01:00
} , {
Name : "copy_cutoff" ,
2021-08-16 11:30:01 +02:00
Help : ` Cutoff for switching to multipart copy .
2019-12-02 18:14:57 +01:00
2020-10-13 23:43:40 +02:00
Any files larger than this that need to be server - side copied will be
2019-12-02 18:14:57 +01:00
copied in chunks of this size .
2021-03-02 20:11:57 +01:00
The minimum is 0 and the maximum is 5 GiB . ` ,
2019-12-02 18:14:57 +01:00
Default : fs . SizeSuffix ( maxSizeForCopy ) ,
Advanced : true ,
2018-05-14 19:06:57 +02:00
} , {
2020-04-23 20:47:48 +02:00
Name : "disable_checksum" ,
2021-08-16 11:30:01 +02:00
Help : ` Don ' t store MD5 checksum with object metadata .
2020-04-23 20:47:48 +02:00
Normally rclone will calculate the MD5 checksum of the input before
uploading it so it can add it to metadata on the object . This is great
for data integrity checking but can cause long delays for large files
2023-03-21 12:43:35 +01:00
to start uploading . ` ,
2018-05-14 19:06:57 +02:00
Default : false ,
Advanced : true ,
2020-06-26 12:14:40 +02:00
} , {
Name : "shared_credentials_file" ,
2021-08-16 11:30:01 +02:00
Help : ` Path to the shared credentials file .
2020-06-26 12:14:40 +02:00
If env_auth = true then rclone can use a shared credentials file .
If this variable is empty rclone will look for the
"AWS_SHARED_CREDENTIALS_FILE" env variable . If the env value is empty
it will default to the current user ' s home directory .
Linux / OSX : "$HOME/.aws/credentials"
Windows : "%USERPROFILE%\.aws\credentials"
` ,
Advanced : true ,
} , {
Name : "profile" ,
2021-08-16 11:30:01 +02:00
Help : ` Profile to use in the shared credentials file .
2020-06-26 12:14:40 +02:00
If env_auth = true then rclone can use a shared credentials file . This
variable controls which profile is used in that file .
If empty it will default to the environment variable "AWS_PROFILE" or
"default" if that environment variable is also not set .
` ,
Advanced : true ,
2018-05-14 19:06:57 +02:00
} , {
2023-07-06 18:55:53 +02:00
Name : "session_token" ,
Help : "An AWS session token." ,
Advanced : true ,
Sensitive : true ,
2018-05-14 19:06:57 +02:00
} , {
2018-10-01 19:36:15 +02:00
Name : "upload_concurrency" ,
2023-12-01 11:30:44 +01:00
Help : ` Concurrency for multipart uploads and copies .
2018-10-01 19:36:15 +02:00
This is the number of chunks of the same file that are uploaded
2023-12-01 11:30:44 +01:00
concurrently for multipart uploads and copies .
2018-10-01 19:36:15 +02:00
2020-10-13 23:59:13 +02:00
If you are uploading small numbers of large files over high - speed links
2018-10-01 19:36:15 +02:00
and these uploads do not fully utilize your bandwidth , then increasing
this may help to speed up the transfers . ` ,
2018-12-02 18:51:14 +01:00
Default : 4 ,
2018-05-14 19:06:57 +02:00
Advanced : true ,
2018-07-18 17:40:59 +02:00
} , {
2018-10-01 19:36:15 +02:00
Name : "force_path_style" ,
Help : ` If true use path style access if false use virtual hosted style .
If this is true ( the default ) then rclone will use path style access ,
if false then rclone will use virtual path style . See [ the AWS S3
docs ] ( https : //docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
for more info .
2020-10-13 23:43:00 +02:00
Some providers ( e . g . AWS , Aliyun OSS , Netease COS , or Tencent COS ) require this set to
2020-01-05 18:53:45 +01:00
false - rclone will do this automatically based on the provider
2024-07-23 12:34:08 +02:00
setting .
Note that if your bucket isn ' t a valid DNS name , i . e . has '.' or '_' in ,
you ' ll need to set this to true .
` ,
2018-07-18 17:40:59 +02:00
Default : true ,
Advanced : true ,
2018-10-09 14:03:37 +02:00
} , {
Name : "v2_auth" ,
Help : ` If true use v2 authentication .
If this is false ( the default ) then rclone will use v4 authentication .
If it is set then rclone will use v2 authentication .
2020-10-13 23:49:58 +02:00
Use this only if v4 signatures don ' t work , e . g . pre Jewel / v10 CEPH . ` ,
2018-10-09 14:03:37 +02:00
Default : false ,
Advanced : true ,
2023-12-03 01:12:54 +01:00
} , {
Name : "use_dual_stack" ,
Help : ` If true use AWS S3 dual - stack endpoint ( IPv6 support ) .
See [ AWS Docs on Dualstack Endpoints ] ( https : //docs.aws.amazon.com/AmazonS3/latest/userguide/dual-stack-endpoints.html)`,
Default : false ,
Advanced : true ,
2019-04-26 11:19:00 +02:00
} , {
Name : "use_accelerate_endpoint" ,
Provider : "AWS" ,
Help : ` If true use the AWS S3 accelerated endpoint .
See : [ AWS S3 Transfer acceleration ] ( https : //docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration-examples.html)`,
Default : false ,
Advanced : true ,
2019-07-24 11:03:38 +02:00
} , {
Name : "leave_parts_on_error" ,
Provider : "AWS" ,
Help : ` If true avoid calling abort upload on a failure , leaving all successfully uploaded parts on S3 for manual recovery .
It should be set to true for resuming uploads across different sessions .
WARNING : Storing parts of an incomplete multipart upload counts towards space usage on S3 and will add additional costs if not cleaned up .
` ,
Default : false ,
Advanced : true ,
2019-12-26 12:05:00 +01:00
} , {
Name : "list_chunk" ,
Help : ` Size of listing chunk ( response list for each ListObject S3 request ) .
This option is also known as "MaxKeys" , "max-items" , or "page-size" from the AWS S3 specification .
Most services truncate the response list to 1000 objects even if requested more than that .
In AWS S3 this is a global maximum and cannot be changed , see [ AWS S3 ] ( https : //docs.aws.amazon.com/cli/latest/reference/s3/ls.html).
In Ceph , this can be increased with the "rgw list buckets max chunk" option .
` ,
Default : 1000 ,
Advanced : true ,
2021-11-03 19:01:23 +01:00
} , {
Name : "list_version" ,
Help : ` Version of ListObjects to use : 1 , 2 or 0 for auto .
When S3 originally launched it only provided the ListObjects call to
enumerate objects in a bucket .
However in May 2016 the ListObjectsV2 call was introduced . This is
much higher performance and should be used if at all possible .
If set to the default , 0 , rclone will guess according to the provider
set which list objects method to call . If it guesses wrong , then it
may be set manually here .
` ,
Default : 0 ,
Advanced : true ,
2021-11-03 20:40:16 +01:00
} , {
Name : "list_url_encode" ,
Help : ` Whether to url encode listings : true / false / unset
Some providers support URL encoding listings and where this is
available this is more reliable when using control characters in file
names . If this is set to unset ( the default ) then rclone will choose
according to the provider setting what to apply , but you can override
rclone ' s choice here .
` ,
Default : fs . Tristate { } ,
Advanced : true ,
2020-07-22 13:02:17 +02:00
} , {
Name : "no_check_bucket" ,
2021-08-16 11:30:01 +02:00
Help : ` If set , don ' t attempt to check the bucket exists or create it .
2020-07-22 13:02:17 +02:00
This can be useful when trying to minimise the number of transactions
rclone does if you know the bucket exists already .
2021-01-10 18:53:32 +01:00
It can also be needed if the user you are using does not have bucket
creation permissions . Before v1 .52 .0 this would have passed silently
due to a bug .
2021-01-29 12:13:42 +01:00
` ,
Default : false ,
Advanced : true ,
} , {
Name : "no_head" ,
2021-08-16 11:30:01 +02:00
Help : ` If set , don ' t HEAD uploaded objects to check integrity .
2021-01-29 12:13:42 +01:00
This can be useful when trying to minimise the number of transactions
rclone does .
Setting it means that if rclone receives a 200 OK message after
uploading an object with PUT then it will assume that it got uploaded
properly .
In particular it will assume :
- the metadata , including modtime , storage class and content type was as uploaded
- the size was as uploaded
It reads the following items from the response for a single part PUT :
- the MD5SUM
- The uploaded date
For multipart uploads these items aren ' t read .
If an source object of unknown length is uploaded then rclone * * will * * do a
HEAD request .
Setting this flag increases the chance for undetected upload failures ,
in particular an incorrect size , so it isn ' t recommended for normal
operation . In practice the chance of an undetected upload failure is
very small even with this flag .
2020-07-22 13:02:17 +02:00
` ,
Default : false ,
Advanced : true ,
2021-04-28 12:05:54 +02:00
} , {
Name : "no_head_object" ,
2021-09-06 11:41:54 +02:00
Help : ` If set, do not do HEAD before GET when getting objects. ` ,
2021-04-28 12:05:54 +02:00
Default : false ,
Advanced : true ,
2020-01-14 18:33:35 +01:00
} , {
Name : config . ConfigEncoding ,
Help : config . ConfigEncodingHelp ,
Advanced : true ,
2020-01-14 22:51:49 +01:00
// Any UTF-8 character is valid in a key, however it can't handle
// invalid UTF-8 and / have a special meaning.
//
// The SDK can't seem to handle uploading files called '.'
//
// FIXME would be nice to add
// - initial / encoding
// - doubled / encoding
// - trailing / encoding
// so that AWS keys are always valid file names
2020-02-19 11:17:25 +01:00
Default : encoder . EncodeInvalidUtf8 |
2020-01-14 22:51:49 +01:00
encoder . EncodeSlash |
2020-02-19 11:17:25 +01:00
encoder . EncodeDot ,
} , {
Name : "memory_pool_flush_time" ,
2023-08-15 21:38:02 +02:00
Default : fs . Duration ( time . Minute ) ,
2020-02-19 11:17:25 +01:00
Advanced : true ,
2023-08-15 21:38:02 +02:00
Hide : fs . OptionHideBoth ,
Help : ` How often internal memory buffer pools will be flushed. (no longer used) ` ,
2020-02-19 11:17:25 +01:00
} , {
Name : "memory_pool_use_mmap" ,
2023-08-15 21:38:02 +02:00
Default : false ,
2020-02-19 11:17:25 +01:00
Advanced : true ,
2023-08-15 21:38:02 +02:00
Hide : fs . OptionHideBoth ,
Help : ` Whether to use mmap buffers in internal memory pool. (no longer used) ` ,
2020-10-13 18:11:22 +02:00
} , {
Name : "disable_http2" ,
Default : false ,
Advanced : true ,
2021-08-16 11:30:01 +02:00
Help : ` Disable usage of http2 for S3 backends .
2020-10-13 18:11:22 +02:00
There is currently an unsolved issue with the s3 ( specifically minio ) backend
and HTTP / 2. HTTP / 2 is enabled by default for the s3 backend but can be
disabled here . When the issue is solved this flag will be removed .
See : https : //github.com/rclone/rclone/issues/4673, https://github.com/rclone/rclone/issues/3631
` ,
2021-10-14 12:19:38 +02:00
} , {
Name : "download_url" ,
Help : ` Custom endpoint for downloads .
This is usually set to a CloudFront CDN URL as AWS S3 offers
cheaper egress for data downloaded through the CloudFront network . ` ,
Advanced : true ,
2022-09-26 08:59:57 +02:00
} , {
Name : "directory_markers" ,
Default : false ,
Advanced : true ,
2023-04-26 11:59:17 +02:00
Help : ` Upload an empty object with a trailing slash when a new directory is created
2022-09-26 08:59:57 +02:00
2023-04-26 11:59:17 +02:00
Empty folders are unsupported for bucket based remotes , this option creates an empty
object ending with "/" , to persist the folder .
2022-09-26 08:59:57 +02:00
` ,
2022-02-27 16:47:31 +01:00
} , {
Name : "use_multipart_etag" ,
Help : ` Whether to use ETag in multipart uploads for verification
2024-08-03 12:35:32 +02:00
This should be true , false or left unset to use the default for the provider .
` ,
Default : fs . Tristate { } ,
Advanced : true ,
} , {
Name : "use_unsigned_payload" ,
Help : ` Whether to use an unsigned payload in PutObject
Rclone has to avoid the AWS SDK seeking the body when calling
PutObject . The AWS provider can add checksums in the trailer to avoid
seeking but other providers can ' t .
2022-02-27 16:47:31 +01:00
This should be true , false or left unset to use the default for the provider .
` ,
Default : fs . Tristate { } ,
Advanced : true ,
2022-05-03 18:39:01 +02:00
} , {
Name : "use_presigned_request" ,
Help : ` Whether to use a presigned request or PutObject for single part uploads
If this is false rclone will use PutObject from the AWS SDK to upload
an object .
Versions of rclone < 1.59 use presigned requests to upload a single
part object and setting this flag to true will re - enable that
functionality . This shouldn ' t be necessary except in exceptional
circumstances or for testing .
` ,
Default : false ,
Advanced : true ,
2022-07-25 17:06:15 +02:00
} , {
Name : "versions" ,
Help : "Include old versions in directory listings." ,
Default : false ,
Advanced : true ,
2022-07-26 18:58:57 +02:00
} , {
Name : "version_at" ,
Help : ` Show file versions as they were at the specified time .
The parameter should be a date , "2006-01-02" , datetime " 2006 - 01 - 02
15 : 04 : 05 " or a duration for that long ago, eg " 100 d " or " 1 h " .
Note that when using this no file write operations are permitted ,
so you can ' t upload files or delete them .
See [ the time option docs ] ( / docs / # time - option ) for valid formats .
` ,
Default : fs . Time { } ,
Advanced : true ,
2023-11-08 16:29:23 +01:00
} , {
Name : "version_deleted" ,
Help : ` Show deleted file markers when using versions .
This shows deleted file markers in the listing when using versions . These will appear
as 0 size files . The only operation which can be performed on them is deletion .
Deleting a delete marker will reveal the previous version .
Deleted files will always show with a timestamp .
` ,
Default : false ,
Advanced : true ,
2022-07-29 18:01:59 +02:00
} , {
Name : "decompress" ,
Help : ` If set this will decompress gzip encoded objects .
It is possible to upload objects to S3 with "Content-Encoding: gzip"
2022-08-14 04:56:32 +02:00
set . Normally rclone will download these files as compressed objects .
2022-07-29 18:01:59 +02:00
If this flag is set then rclone will decompress these files with
"Content-Encoding: gzip" as they are received . This means that rclone
can ' t check the size and hash but the file contents will be decompressed .
` ,
Advanced : true ,
Default : false ,
2022-10-29 00:23:29 +02:00
} , {
Name : "might_gzip" ,
Help : strings . ReplaceAll ( ` Set this if the backend might gzip objects .
Normally providers will not alter objects when they are downloaded . If
an object was not uploaded with | Content - Encoding : gzip | then it won ' t
be set on download .
However some providers may gzip objects even if they weren ' t uploaded
with | Content - Encoding : gzip | ( eg Cloudflare ) .
A symptom of this would be receiving errors like
ERROR corrupted on transfer : sizes differ NNN vs MMM
If you set this flag and rclone downloads an object with
Content - Encoding : gzip set and chunked transfer encoding , then rclone
will decompress the object on the fly .
If this is set to unset ( the default ) then rclone will choose
according to the provider setting what to apply , but you can override
rclone ' s choice here .
2023-01-09 07:54:51 +01:00
` , "|", " ` " ) ,
Default : fs . Tristate { } ,
Advanced : true ,
} , {
Name : "use_accept_encoding_gzip" ,
Help : strings . ReplaceAll ( ` Whether to send | Accept - Encoding : gzip | header .
By default , rclone will append | Accept - Encoding : gzip | to the request to download
compressed objects whenever possible .
However some providers such as Google Cloud Storage may alter the HTTP headers , breaking
the signature of the request .
A symptom of this would be receiving errors like
SignatureDoesNotMatch : The request signature we calculated does not match the signature you provided .
In this case , you might want to try disabling this option .
2022-10-29 00:23:29 +02:00
` , "|", " ` " ) ,
Default : fs . Tristate { } ,
Advanced : true ,
2022-10-12 09:55:58 +02:00
} , {
Name : "no_system_metadata" ,
Help : ` Suppress setting and reading of system metadata ` ,
Advanced : true ,
Default : false ,
2023-03-02 10:56:09 +01:00
} , {
Name : "sts_endpoint" ,
2024-08-03 12:35:32 +02:00
Help : "Endpoint for STS (deprecated).\n\nLeave blank if using AWS to use the default endpoint for the region." ,
2023-03-02 10:56:09 +01:00
Provider : "AWS" ,
Advanced : true ,
2024-08-03 12:35:32 +02:00
Hide : fs . OptionHideBoth ,
2023-10-06 12:45:03 +02:00
} , {
Name : "use_already_exists" ,
Help : strings . ReplaceAll ( ` Set if rclone should report BucketAlreadyExists errors on bucket creation .
At some point during the evolution of the s3 protocol , AWS started
returning an | AlreadyOwnedByYou | error when attempting to create a
bucket that the user already owned , rather than a
| BucketAlreadyExists | error .
Unfortunately exactly what has been implemented by s3 clones is a
little inconsistent , some return | AlreadyOwnedByYou | , some return
| BucketAlreadyExists | and some return no error at all .
This is important to rclone because it ensures the bucket exists by
creating it on quite a lot of operations ( unless
| -- s3 - no - check - bucket | is used ) .
If rclone knows the provider can return | AlreadyOwnedByYou | or returns
no error then it can report | BucketAlreadyExists | errors when the user
attempts to create a bucket not owned by them . Otherwise rclone
ignores the | BucketAlreadyExists | error which can lead to confusion .
This should be automatically set correctly for all providers rclone
knows about - please make a bug report if not .
` , "|", " ` " ) ,
Default : fs . Tristate { } ,
Advanced : true ,
2023-11-14 13:39:50 +01:00
} , {
Name : "use_multipart_uploads" ,
Help : ` Set if rclone should use multipart uploads .
You can change this if you want to disable the use of multipart uploads .
This shouldn ' t be necessary in normal operation .
This should be automatically set correctly for all providers rclone
knows about - please make a bug report if not .
` ,
Default : fs . Tristate { } ,
Advanced : true ,
2024-08-06 11:33:17 +02:00
} , {
Name : "sdk_log_mode" ,
Help : strings . ReplaceAll ( ` Set to debug the SDK
This can be set to a comma separated list of the following functions :
- | Signing |
- | Retries |
- | Request |
- | RequestWithBody |
- | Response |
- | ResponseWithBody |
- | DeprecatedUsage |
- | RequestEventMessage |
- | ResponseEventMessage |
Use | Off | to disable and | All | to set all log levels . You will need to
use | - vv | to see the debug level logs .
` , "|", " ` " ) ,
Default : sdkLogMode ( 0 ) ,
Advanced : true ,
2020-02-19 11:17:25 +01:00
} ,
} } )
2013-06-27 21:13:07 +02:00
}
2013-01-08 19:53:35 +01:00
// Constants
const (
2022-05-24 13:32:39 +02:00
metaMtime = "mtime" // the meta key to store mtime in - e.g. X-Amz-Meta-Mtime
metaMD5Hash = "md5chksum" // the meta key to store md5hash in
2021-03-02 20:11:57 +01:00
// The maximum size of object we can COPY - this should be 5 GiB but is < 5 GB for b2 compatibility
2020-09-01 19:53:29 +02:00
// See https://forum.rclone.org/t/copying-files-within-a-b2-bucket/16680/76
maxSizeForCopy = 4768 * 1024 * 1024
maxUploadParts = 10000 // maximum allowed number of parts in a multi-part upload
2019-12-31 00:17:06 +01:00
minChunkSize = fs . SizeSuffix ( 1024 * 1024 * 5 )
2018-11-26 22:09:23 +01:00
defaultUploadCutoff = fs . SizeSuffix ( 200 * 1024 * 1024 )
maxUploadCutoff = fs . SizeSuffix ( 5 * 1024 * 1024 * 1024 )
2023-08-15 21:38:02 +02:00
minSleep = 10 * time . Millisecond // In case of error, start at 10ms sleep.
2020-06-18 18:50:50 +02:00
maxExpireDuration = fs . Duration ( 7 * 24 * time . Hour ) // max expiry is 1 week
2013-01-08 19:53:35 +01:00
)
2024-08-06 11:33:17 +02:00
type (
sdkLogMode = fs . Bits [ sdkLogModeChoices ]
sdkLogModeChoices struct { }
)
func ( sdkLogModeChoices ) Choices ( ) [ ] fs . BitsChoicesInfo {
return [ ] fs . BitsChoicesInfo {
{ Bit : uint64 ( 0 ) , Name : "Off" } ,
{ Bit : uint64 ( aws . LogSigning ) , Name : "Signing" } ,
{ Bit : uint64 ( aws . LogRetries ) , Name : "Retries" } ,
{ Bit : uint64 ( aws . LogRequest ) , Name : "Request" } ,
{ Bit : uint64 ( aws . LogRequestWithBody ) , Name : "RequestWithBody" } ,
{ Bit : uint64 ( aws . LogResponse ) , Name : "Response" } ,
{ Bit : uint64 ( aws . LogResponseWithBody ) , Name : "ResponseWithBody" } ,
{ Bit : uint64 ( aws . LogDeprecatedUsage ) , Name : "DeprecatedUsage" } ,
{ Bit : uint64 ( aws . LogRequestEventMessage ) , Name : "RequestEventMessage" } ,
{ Bit : uint64 ( aws . LogResponseEventMessage ) , Name : "ResponseEventMessage" } ,
{ Bit : math . MaxUint64 , Name : "All" } ,
}
}
2022-07-26 18:58:57 +02:00
// globals
var (
errNotWithVersionAt = errors . New ( "can't modify or delete files in --s3-version-at mode" )
)
2022-05-24 13:32:39 +02:00
// system metadata keys which this backend owns
var systemMetadataInfo = map [ string ] fs . MetadataHelp {
"cache-control" : {
Help : "Cache-Control header" ,
Type : "string" ,
Example : "no-cache" ,
} ,
"content-disposition" : {
Help : "Content-Disposition header" ,
Type : "string" ,
Example : "inline" ,
} ,
"content-encoding" : {
Help : "Content-Encoding header" ,
Type : "string" ,
Example : "gzip" ,
} ,
"content-language" : {
Help : "Content-Language header" ,
Type : "string" ,
Example : "en-US" ,
} ,
"content-type" : {
Help : "Content-Type header" ,
Type : "string" ,
Example : "text/plain" ,
} ,
// "tagging": {
// Help: "x-amz-tagging header",
// Type: "string",
// Example: "tag1=value1&tag2=value2",
// },
"tier" : {
Help : "Tier of the object" ,
Type : "string" ,
Example : "GLACIER" ,
ReadOnly : true ,
} ,
"mtime" : {
Help : "Time of last modification, read from rclone metadata" ,
Type : "RFC 3339" ,
Example : "2006-01-02T15:04:05.999999999Z07:00" ,
} ,
"btime" : {
Help : "Time of file birth (creation) read from Last-Modified header" ,
Type : "RFC 3339" ,
Example : "2006-01-02T15:04:05.999999999Z07:00" ,
ReadOnly : true ,
} ,
}
2018-05-14 19:06:57 +02:00
// Options defines the configuration for this backend
type Options struct {
2020-01-14 18:33:35 +01:00
Provider string ` config:"provider" `
EnvAuth bool ` config:"env_auth" `
AccessKeyID string ` config:"access_key_id" `
SecretAccessKey string ` config:"secret_access_key" `
Region string ` config:"region" `
Endpoint string ` config:"endpoint" `
2023-03-02 10:56:09 +01:00
STSEndpoint string ` config:"sts_endpoint" `
2023-12-03 01:12:54 +01:00
UseDualStack bool ` config:"use_dual_stack" `
2020-01-14 18:33:35 +01:00
LocationConstraint string ` config:"location_constraint" `
ACL string ` config:"acl" `
BucketACL string ` config:"bucket_acl" `
2020-12-03 03:30:06 +01:00
RequesterPays bool ` config:"requester_pays" `
2020-01-14 18:33:35 +01:00
ServerSideEncryption string ` config:"server_side_encryption" `
SSEKMSKeyID string ` config:"sse_kms_key_id" `
2020-03-30 12:26:52 +02:00
SSECustomerAlgorithm string ` config:"sse_customer_algorithm" `
SSECustomerKey string ` config:"sse_customer_key" `
2022-09-17 18:28:44 +02:00
SSECustomerKeyBase64 string ` config:"sse_customer_key_base64" `
2020-03-30 12:26:52 +02:00
SSECustomerKeyMD5 string ` config:"sse_customer_key_md5" `
2020-01-14 18:33:35 +01:00
StorageClass string ` config:"storage_class" `
UploadCutoff fs . SizeSuffix ` config:"upload_cutoff" `
CopyCutoff fs . SizeSuffix ` config:"copy_cutoff" `
ChunkSize fs . SizeSuffix ` config:"chunk_size" `
2022-08-09 11:44:54 +02:00
MaxUploadParts int ` config:"max_upload_parts" `
2020-01-14 18:33:35 +01:00
DisableChecksum bool ` config:"disable_checksum" `
2020-06-26 12:14:40 +02:00
SharedCredentialsFile string ` config:"shared_credentials_file" `
Profile string ` config:"profile" `
2020-01-14 18:33:35 +01:00
SessionToken string ` config:"session_token" `
UploadConcurrency int ` config:"upload_concurrency" `
ForcePathStyle bool ` config:"force_path_style" `
V2Auth bool ` config:"v2_auth" `
UseAccelerateEndpoint bool ` config:"use_accelerate_endpoint" `
LeavePartsOnError bool ` config:"leave_parts_on_error" `
2024-08-03 12:35:32 +02:00
ListChunk int32 ` config:"list_chunk" `
2021-11-03 19:01:23 +01:00
ListVersion int ` config:"list_version" `
2021-11-03 20:40:16 +01:00
ListURLEncode fs . Tristate ` config:"list_url_encode" `
2020-07-22 13:02:17 +02:00
NoCheckBucket bool ` config:"no_check_bucket" `
2021-01-29 12:13:42 +01:00
NoHead bool ` config:"no_head" `
2021-04-28 12:05:54 +02:00
NoHeadObject bool ` config:"no_head_object" `
2020-01-14 18:33:35 +01:00
Enc encoder . MultiEncoder ` config:"encoding" `
2020-10-13 18:11:22 +02:00
DisableHTTP2 bool ` config:"disable_http2" `
2021-10-14 12:19:38 +02:00
DownloadURL string ` config:"download_url" `
2022-09-26 08:59:57 +02:00
DirectoryMarkers bool ` config:"directory_markers" `
2022-02-27 16:47:31 +01:00
UseMultipartEtag fs . Tristate ` config:"use_multipart_etag" `
2022-05-03 18:39:01 +02:00
UsePresignedRequest bool ` config:"use_presigned_request" `
2022-07-25 17:06:15 +02:00
Versions bool ` config:"versions" `
2022-07-26 18:58:57 +02:00
VersionAt fs . Time ` config:"version_at" `
2023-11-08 16:29:23 +01:00
VersionDeleted bool ` config:"version_deleted" `
2022-07-29 18:01:59 +02:00
Decompress bool ` config:"decompress" `
2022-10-29 00:23:29 +02:00
MightGzip fs . Tristate ` config:"might_gzip" `
2023-01-09 07:54:51 +01:00
UseAcceptEncodingGzip fs . Tristate ` config:"use_accept_encoding_gzip" `
2022-10-12 09:55:58 +02:00
NoSystemMetadata bool ` config:"no_system_metadata" `
2023-10-06 12:45:03 +02:00
UseAlreadyExists fs . Tristate ` config:"use_already_exists" `
2023-11-14 13:39:50 +01:00
UseMultipartUploads fs . Tristate ` config:"use_multipart_uploads" `
2024-08-03 12:35:32 +02:00
UseUnsignedPayload fs . Tristate ` config:"use_unsigned_payload" `
2024-08-06 11:33:17 +02:00
SDKLogMode sdkLogMode ` config:"sdk_log_mode" `
2018-05-14 19:06:57 +02:00
}
2016-09-01 23:27:50 +02:00
2015-11-07 12:14:46 +01:00
// Fs represents a remote s3 server
type Fs struct {
2024-08-03 12:35:32 +02:00
name string // the name of the remote
root string // root of the bucket - ignore all objects above this
opt Options // parsed options
ci * fs . ConfigInfo // global config
ctx context . Context // global context for reading config
features * fs . Features // optional features
c * s3 . Client // the connection to the s3 server
rootBucket string // bucket part of root (if any)
rootDirectory string // directory part of root (if any)
cache * bucket . Cache // cache for bucket creation status
pacer * fs . Pacer // To pace the API calls
srv * http . Client // a plain http client
srvRest * rest . Client // the rest connection to the server
etagIsNotMD5 bool // if set ETags are not MD5s
2022-07-29 18:01:59 +02:00
versioningMu sync . Mutex
versioning fs . Tristate // if set bucket is using versions
warnCompressed sync . Once // warn once about compressed files
2013-01-08 19:53:35 +01:00
}
2015-11-07 12:14:46 +01:00
// Object describes a s3 object
type Object struct {
2013-01-08 19:53:35 +01:00
// Will definitely have everything but meta which may be nil
//
2016-09-21 23:13:24 +02:00
// List will read everything but meta & mimeType - to fill
// that in you need to call readMetaData
2022-06-22 16:40:30 +02:00
fs * Fs // what this object is part of
remote string // The remote path
md5 string // md5sum of the object
bytes int64 // size of the object
lastModified time . Time // Last modified
meta map [ string ] string // The object metadata if known - may be nil - with lower case keys
mimeType string // MimeType of object - may be ""
2022-07-25 17:06:15 +02:00
versionID * string // If present this points to an object version
2022-05-24 13:32:39 +02:00
// Metadata as pointers to strings as they often won't be present
storageClass * string // e.g. GLACIER
cacheControl * string // Cache-Control: header
contentDisposition * string // Content-Disposition: header
contentEncoding * string // Content-Encoding: header
contentLanguage * string // Content-Language: header
2013-01-08 19:53:35 +01:00
}
2024-08-03 12:35:32 +02:00
// safely dereference the pointer, returning a zero T if nil
func deref [ T any ] ( p * T ) T {
if p == nil {
var zero T
return zero
}
return * p
}
// gets an http status code from err or returns -1
func getHTTPStatusCode ( err error ) int {
var httpErr interface { HTTPStatusCode ( ) int }
if errors . As ( err , & httpErr ) {
return httpErr . HTTPStatusCode ( )
}
return - 1
}
2013-01-08 19:53:35 +01:00
// ------------------------------------------------------------
2015-09-22 19:47:16 +02:00
// Name of the remote (as passed into NewFs)
2015-11-07 12:14:46 +01:00
func ( f * Fs ) Name ( ) string {
2015-08-22 17:53:11 +02:00
return f . name
}
2015-09-22 19:47:16 +02:00
// Root of the remote (as passed into NewFs)
2015-11-07 12:14:46 +01:00
func ( f * Fs ) Root ( ) string {
2019-08-09 12:29:36 +02:00
return f . root
2015-09-01 21:45:27 +02:00
}
2015-11-07 12:14:46 +01:00
// String converts this Fs to a string
func ( f * Fs ) String ( ) string {
2019-08-09 12:29:36 +02:00
if f . rootBucket == "" {
2022-06-08 22:25:17 +02:00
return "S3 root"
2019-08-09 12:29:36 +02:00
}
if f . rootDirectory == "" {
return fmt . Sprintf ( "S3 bucket %s" , f . rootBucket )
2014-05-05 19:25:32 +02:00
}
2019-08-09 12:29:36 +02:00
return fmt . Sprintf ( "S3 bucket %s path %s" , f . rootBucket , f . rootDirectory )
2013-01-08 19:53:35 +01:00
}
2017-01-13 18:21:47 +01:00
// Features returns the optional features of this Fs
func ( f * Fs ) Features ( ) * fs . Features {
return f . features
}
2018-09-03 06:41:04 +02:00
// retryErrorCodes is a slice of error codes that we will retry
// See: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
var retryErrorCodes = [ ] int {
2020-12-01 15:25:25 +01:00
429 , // Too Many Requests
2020-02-11 16:30:32 +01:00
500 , // Internal Server Error - "We encountered an internal error. Please try again."
2018-09-03 06:41:04 +02:00
503 , // Service Unavailable/Slow Down - "Reduce your request rate"
}
2022-08-05 17:35:41 +02:00
// S3 is pretty resilient, and the built in retry handling is probably sufficient
2018-09-03 06:41:04 +02:00
// as it should notice closed connections and timeouts which are the most likely
// sort of failure modes
2021-03-16 16:50:02 +01:00
func ( f * Fs ) shouldRetry ( ctx context . Context , err error ) ( bool , error ) {
if fserrors . ContextError ( ctx , & err ) {
return false , err
}
2024-08-03 12:35:32 +02:00
// https://github.com/aws/aws-sdk-go-v2/blob/main/CHANGELOG.md#error-handling
2018-09-03 06:41:04 +02:00
// If this is an awserr object, try and extract more useful information to determine if we should retry
2024-08-03 12:35:32 +02:00
var awsError smithy . APIError
if errors . As ( err , & awsError ) {
2019-02-07 18:41:17 +01:00
// Simple case, check the original embedded error in case it's generically retryable
2024-08-03 12:35:32 +02:00
if fserrors . ShouldRetry ( awsError ) {
2018-09-03 06:41:04 +02:00
return true , err
}
2021-11-24 13:48:57 +01:00
// If it is a timeout then we want to retry that
2024-08-03 12:35:32 +02:00
if awsError . ErrorCode ( ) == "RequestTimeout" {
2021-11-24 13:48:57 +01:00
return true , err
}
2024-08-03 12:35:32 +02:00
}
// Check http status code if available
if httpStatusCode := getHTTPStatusCode ( err ) ; httpStatusCode > 0 {
// 301 if wrong region for bucket - can only update if running from a bucket
if f . rootBucket != "" {
if httpStatusCode == http . StatusMovedPermanently {
urfbErr := f . updateRegionForBucket ( ctx , f . rootBucket )
if urfbErr != nil {
fs . Errorf ( f , "Failed to update region for bucket: %v" , urfbErr )
return false , err
2019-01-16 14:35:19 +01:00
}
2024-08-03 12:35:32 +02:00
return true , err
2019-01-16 14:35:19 +01:00
}
2024-08-03 12:35:32 +02:00
}
for _ , e := range retryErrorCodes {
if httpStatusCode == e {
return true , err
2018-09-03 06:41:04 +02:00
}
}
}
2019-01-12 17:46:45 +01:00
// Ok, not an awserr, check for generic failure conditions
2018-09-03 06:41:04 +02:00
return fserrors . ShouldRetry ( err ) , err
}
2019-08-09 12:29:36 +02:00
// parsePath parses a remote 'url'
func parsePath ( path string ) ( root string ) {
root = strings . Trim ( path , "/" )
2013-01-08 19:53:35 +01:00
return
}
2019-08-09 12:29:36 +02:00
// split returns bucket and bucketPath from the rootRelativePath
// relative to f.root
func ( f * Fs ) split ( rootRelativePath string ) ( bucketName , bucketPath string ) {
2023-03-03 11:45:29 +01:00
bucketName , bucketPath = bucket . Split ( bucket . Join ( f . root , rootRelativePath ) )
2020-01-14 18:33:35 +01:00
return f . opt . Enc . FromStandardName ( bucketName ) , f . opt . Enc . FromStandardPath ( bucketPath )
2019-08-09 12:29:36 +02:00
}
// split returns bucket and bucketPath from the object
func ( o * Object ) split ( ) ( bucket , bucketPath string ) {
2022-07-25 17:06:15 +02:00
bucket , bucketPath = o . fs . split ( o . remote )
// If there is an object version, then the path may have a
// version suffix, if so remove it.
//
// If we are unlucky enough to have a file name with a valid
// version path where this wasn't required (eg using
// --s3-version-at) then this will go wrong.
if o . versionID != nil {
_ , bucketPath = version . Remove ( bucketPath )
}
return bucket , bucketPath
2019-08-09 12:29:36 +02:00
}
2020-10-13 18:11:22 +02:00
// getClient makes an http client according to the options
2020-11-05 12:33:32 +01:00
func getClient ( ctx context . Context , opt * Options ) * http . Client {
2020-10-13 18:11:22 +02:00
// TODO: Do we need cookies too?
2020-11-13 16:24:43 +01:00
t := fshttp . NewTransportCustom ( ctx , func ( t * http . Transport ) {
2020-10-13 18:11:22 +02:00
if opt . DisableHTTP2 {
t . TLSNextProto = map [ string ] func ( string , * tls . Conn ) http . RoundTripper { }
}
} )
return & http . Client {
Transport : t ,
}
}
2024-08-05 16:48:29 +02:00
// Google Cloud Storage alters the Accept-Encoding header, which
// breaks the v2 request signature
//
// It also doesn't like the x-id URL parameter SDKv2 puts in so we
// remove that too.
//
// See https://github.com/aws/aws-sdk-go-v2/issues/1816.
// Adapted from: https://github.com/aws/aws-sdk-go-v2/issues/1816#issuecomment-1927281540
func fixupGCS ( o * s3 . Options ) {
type ignoredHeadersKey struct { }
headers := [ ] string { "Accept-Encoding" }
fixup := middleware . FinalizeMiddlewareFunc (
"FixupGCS" ,
func ( ctx context . Context , in middleware . FinalizeInput , next middleware . FinalizeHandler ) ( out middleware . FinalizeOutput , metadata middleware . Metadata , err error ) {
req , ok := in . Request . ( * smithyhttp . Request )
if ! ok {
return out , metadata , fmt . Errorf ( "fixupGCS: unexpected request middleware type %T" , in . Request )
}
// Delete headers from being signed - will restore later
ignored := make ( map [ string ] string , len ( headers ) )
for _ , h := range headers {
ignored [ h ] = req . Header . Get ( h )
req . Header . Del ( h )
}
// Remove x-id because Google doesn't like them
if query := req . URL . Query ( ) ; query . Has ( "x-id" ) {
query . Del ( "x-id" )
req . URL . RawQuery = query . Encode ( )
}
// Store ignored on context
ctx = middleware . WithStackValue ( ctx , ignoredHeadersKey { } , ignored )
return next . HandleFinalize ( ctx , in )
} ,
)
// Restore headers if necessary
restore := middleware . FinalizeMiddlewareFunc (
"FixupGCSRestoreHeaders" ,
func ( ctx context . Context , in middleware . FinalizeInput , next middleware . FinalizeHandler ) ( out middleware . FinalizeOutput , metadata middleware . Metadata , err error ) {
req , ok := in . Request . ( * smithyhttp . Request )
if ! ok {
return out , metadata , fmt . Errorf ( "fixupGCS: unexpected request middleware type %T" , in . Request )
}
// Restore ignored from ctx
ignored , _ := middleware . GetStackValue ( ctx , ignoredHeadersKey { } ) . ( map [ string ] string )
for k , v := range ignored {
req . Header . Set ( k , v )
}
return next . HandleFinalize ( ctx , in )
} ,
)
o . APIOptions = append ( o . APIOptions , func ( stack * middleware . Stack ) error {
if err := stack . Finalize . Insert ( fixup , "Signing" , middleware . Before ) ; err != nil {
return err
}
if err := stack . Finalize . Insert ( restore , "Signing" , middleware . After ) ; err != nil {
return err
}
return nil
} )
}
2024-08-06 11:33:17 +02:00
// A logger for the S3 SDK
type s3logger struct { }
// Logf is expected to support the standard fmt package "verbs".
func ( s3logger ) Logf ( classification logging . Classification , format string , v ... interface { } ) {
switch classification {
default :
case logging . Debug :
fs . Debugf ( "S3 SDK" , format , v ... )
case logging . Warn :
fs . Infof ( "S3 SDK" , format , v ... )
}
}
2013-01-08 19:53:35 +01:00
// s3Connection makes a connection to s3
2024-08-03 12:35:32 +02:00
func s3Connection ( ctx context . Context , opt * Options , client * http . Client ) ( s3Client * s3 . Client , err error ) {
2021-08-13 18:27:49 +02:00
ci := fs . GetConfig ( ctx )
2024-08-03 12:35:32 +02:00
var awsConfig aws . Config
2017-11-22 22:21:36 +01:00
2024-08-03 12:35:32 +02:00
// Try to fill in the config from the environment if env_auth=true
if opt . EnvAuth {
configOpts := [ ] func ( * awsconfig . LoadOptions ) error { }
// Set the name of the profile if supplied
if opt . Profile != "" {
configOpts = append ( configOpts , awsconfig . WithSharedConfigProfile ( opt . Profile ) )
}
// Set the shared config file if supplied
if opt . SharedCredentialsFile != "" {
configOpts = append ( configOpts , awsconfig . WithSharedConfigFiles ( [ ] string { opt . SharedCredentialsFile } ) )
}
awsConfig , err = awsconfig . LoadDefaultConfig ( ctx , configOpts ... )
if err != nil {
return nil , fmt . Errorf ( "couldn't load configuration with env_auth=true: %w" , err )
}
} else {
switch {
case opt . AccessKeyID == "" && opt . SecretAccessKey == "" :
// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
awsConfig . Credentials = aws . AnonymousCredentials { }
fs . Debugf ( nil , "Using anonymous credentials - did you mean to set env_auth=true?" )
case opt . AccessKeyID == "" :
return nil , errors . New ( "access_key_id not found" )
case opt . SecretAccessKey == "" :
return nil , errors . New ( "secret_access_key not found" )
default :
// Make the static auth
v := aws . Credentials {
AccessKeyID : opt . AccessKeyID ,
SecretAccessKey : opt . SecretAccessKey ,
SessionToken : opt . SessionToken ,
}
awsConfig . Credentials = & credentials . StaticCredentialsProvider { Value : v }
}
2013-01-08 19:53:35 +01:00
}
2018-05-14 19:06:57 +02:00
if opt . Region == "" {
opt . Region = "us-east-1"
2013-01-08 19:53:35 +01:00
}
2021-11-03 19:01:23 +01:00
setQuirks ( opt )
2024-08-03 12:35:32 +02:00
awsConfig . RetryMaxAttempts = ci . LowLevelRetries
awsConfig . HTTPClient = client
options := [ ] func ( * s3 . Options ) { }
options = append ( options , func ( s3Opt * s3 . Options ) {
s3Opt . UsePathStyle = opt . ForcePathStyle
s3Opt . UseAccelerate = opt . UseAccelerateEndpoint
// FIXME maybe this should be a tristate so can default to DualStackEndpointStateUnset?
if opt . UseDualStack {
s3Opt . EndpointOptions . UseDualStackEndpoint = aws . DualStackEndpointStateEnabled
} else {
s3Opt . EndpointOptions . UseDualStackEndpoint = aws . DualStackEndpointStateDisabled
}
// FIXME not ported from SDK v1 - not sure what this does
// s3Opt.UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint
} )
2020-06-26 15:09:29 +02:00
2019-01-16 14:35:19 +01:00
if opt . Region != "" {
2024-08-03 12:35:32 +02:00
awsConfig . Region = opt . Region
2019-01-16 14:35:19 +01:00
}
2024-08-03 12:35:32 +02:00
if opt . STSEndpoint != "" {
// FIXME not sure if anyone is using this
// Haven't figured out how to do it with the v2 SDK
return nil , errors . New ( "--s3-sts-endpoint is no longer supported with the v2 SDK - please make an issue" )
2019-01-16 14:35:19 +01:00
}
2024-08-03 12:35:32 +02:00
if opt . Endpoint != "" {
if ! strings . HasPrefix ( opt . Endpoint , "http" ) {
opt . Endpoint = "https://" + opt . Endpoint
2021-10-29 13:39:34 +02:00
}
2024-08-03 12:35:32 +02:00
options = append ( options , func ( s3Opt * s3 . Options ) {
s3Opt . BaseEndpoint = & opt . Endpoint
} )
2022-06-16 23:55:45 +02:00
}
2024-08-03 12:35:32 +02:00
2022-06-16 23:55:45 +02:00
if opt . V2Auth || opt . Region == "other-v2-signature" {
fs . Debugf ( nil , "Using v2 auth" )
2024-08-03 12:35:32 +02:00
options = append ( options , func ( s3Opt * s3 . Options ) {
s3Opt . HTTPSignerV4 = & v2Signer { opt : opt }
} )
2013-01-08 19:53:35 +01:00
}
2024-08-03 12:35:32 +02:00
2024-08-05 16:48:29 +02:00
if opt . Provider == "GCS" {
options = append ( options , func ( o * s3 . Options ) {
fixupGCS ( o )
} )
}
2024-08-06 11:33:17 +02:00
// Enable SDK logging if requested
if opt . SDKLogMode != 0 {
awsConfig . ClientLogMode = aws . ClientLogMode ( opt . SDKLogMode )
awsConfig . Logger = s3logger { }
}
2024-08-03 12:35:32 +02:00
c := s3 . NewFromConfig ( awsConfig , options ... )
return c , nil
2013-01-08 19:53:35 +01:00
}
2018-09-07 13:02:27 +02:00
func checkUploadChunkSize ( cs fs . SizeSuffix ) error {
if cs < minChunkSize {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "%s is less than %s" , cs , minChunkSize )
2018-09-07 13:02:27 +02:00
}
return nil
}
func ( f * Fs ) setUploadChunkSize ( cs fs . SizeSuffix ) ( old fs . SizeSuffix , err error ) {
err = checkUploadChunkSize ( cs )
if err == nil {
old , f . opt . ChunkSize = f . opt . ChunkSize , cs
}
return
}
2024-03-13 12:53:38 +01:00
func checkCopyCutoff ( cs fs . SizeSuffix ) error {
minCopySize := fs . SizeSuffixBase
if cs < minCopySize {
return fmt . Errorf ( "value is too small (%v is less than %v)" , cs , minCopySize )
}
return nil
}
2018-11-26 22:09:23 +01:00
func checkUploadCutoff ( cs fs . SizeSuffix ) error {
if cs > maxUploadCutoff {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "%s is greater than %s" , cs , maxUploadCutoff )
2018-11-26 22:09:23 +01:00
}
return nil
}
func ( f * Fs ) setUploadCutoff ( cs fs . SizeSuffix ) ( old fs . SizeSuffix , err error ) {
2023-11-24 17:32:06 +01:00
if f . opt . Provider != "Rclone" {
err = checkUploadCutoff ( cs )
}
2018-11-26 22:09:23 +01:00
if err == nil {
old , f . opt . UploadCutoff = f . opt . UploadCutoff , cs
}
return
}
2023-11-24 13:36:48 +01:00
func ( f * Fs ) setCopyCutoff ( cs fs . SizeSuffix ) ( old fs . SizeSuffix , err error ) {
2024-08-06 12:50:21 +02:00
if f . opt . CopyCutoff == math . MaxInt64 {
return f . opt . CopyCutoff , fmt . Errorf ( "--s3-copy-cutoff not supported: %w" , fs . ErrorNotImplemented )
}
2023-11-24 13:36:48 +01:00
err = checkUploadChunkSize ( cs )
if err == nil {
old , f . opt . CopyCutoff = f . opt . CopyCutoff , cs
}
return
}
2022-06-18 09:29:21 +02:00
// setEndpointValueForIDriveE2 gets user region endpoint against the Access Key details by calling the API
func setEndpointValueForIDriveE2 ( m configmap . Mapper ) ( err error ) {
value , ok := m . Get ( fs . ConfigProvider )
if ! ok || value != "IDrive" {
return
}
value , ok = m . Get ( "access_key_id" )
if ! ok || value == "" {
return
}
client := & http . Client { Timeout : time . Second * 3 }
// API to get user region endpoint against the Access Key details: https://www.idrive.com/e2/guides/get_region_endpoint
resp , err := client . Post ( "https://api.idrivee2.com/api/service/get_region_end_point" ,
"application/json" ,
strings . NewReader ( ` { "access_key": " ` + value + ` "} ` ) )
if err != nil {
return
}
defer fs . CheckClose ( resp . Body , & err )
decoder := json . NewDecoder ( resp . Body )
var data = & struct {
RespCode int ` json:"resp_code" `
RespMsg string ` json:"resp_msg" `
DomainName string ` json:"domain_name" `
} { }
if err = decoder . Decode ( data ) ; err == nil && data . RespCode == 0 {
m . Set ( "endpoint" , data . DomainName )
}
return
}
2021-11-03 19:01:23 +01:00
// Set the provider quirks
//
2021-11-03 20:40:16 +01:00
// There should be no testing against opt.Provider anywhere in the
// code except in here to localise the setting of the quirks.
//
2023-09-21 13:35:23 +02:00
// Run the integration tests to check you have the quirks correct.
//
// go test -v -remote NewS3Provider:
2021-11-03 19:01:23 +01:00
func setQuirks ( opt * Options ) {
var (
2023-09-21 13:35:23 +02:00
listObjectsV2 = true // Always use ListObjectsV2 instead of ListObjects
virtualHostStyle = true // Use bucket.provider.com instead of putting the bucket in the URL
urlEncodeListings = true // URL encode the listings to help with control characters
useMultipartEtag = true // Set if Etags for multpart uploads are compatible with AWS
useAcceptEncodingGzip = true // Set Accept-Encoding: gzip
mightGzip = true // assume all providers might use content encoding gzip until proven otherwise
2023-10-06 12:45:03 +02:00
useAlreadyExists = true // Set if provider returns AlreadyOwnedByYou or no error if you try to remake your own bucket
2023-11-14 13:39:50 +01:00
useMultipartUploads = true // Set if provider supports multipart uploads
2024-08-03 12:35:32 +02:00
useUnsignedPayload = true // Do we need to use unsigned payloads to avoid seeking in PutObject
2021-11-03 19:01:23 +01:00
)
switch opt . Provider {
case "AWS" :
// No quirks
2024-08-03 12:35:32 +02:00
mightGzip = false // Never auto gzips objects
useUnsignedPayload = false // AWS has trailer support which means it adds checksums in the trailer without seeking
2021-11-03 19:01:23 +01:00
case "Alibaba" :
2022-02-27 16:47:31 +01:00
useMultipartEtag = false // Alibaba seems to calculate multipart Etags differently from AWS
2023-10-06 12:45:03 +02:00
useAlreadyExists = true // returns 200 OK
2022-06-07 09:41:46 +02:00
case "HuaweiOBS" :
// Huawei OBS PFS is not support listObjectV2, and if turn on the urlEncodeListing, marker will not work and keep list same page forever.
urlEncodeListings = false
listObjectsV2 = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2021-11-03 19:01:23 +01:00
case "Ceph" :
2021-11-03 20:40:16 +01:00
listObjectsV2 = false
virtualHostStyle = false
urlEncodeListings = false
2024-06-05 13:32:38 +02:00
useAlreadyExists = true
2022-03-24 12:57:00 +01:00
case "ChinaMobile" :
listObjectsV2 = false
virtualHostStyle = false
urlEncodeListings = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2022-05-06 13:09:29 +02:00
case "Cloudflare" :
virtualHostStyle = false
useMultipartEtag = false // currently multipart Etags are random
2022-04-28 11:42:30 +02:00
case "ArvanCloud" :
listObjectsV2 = false
virtualHostStyle = false
urlEncodeListings = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2021-11-03 19:01:23 +01:00
case "DigitalOcean" :
2021-11-03 20:40:16 +01:00
urlEncodeListings = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2021-11-03 19:01:23 +01:00
case "Dreamhost" :
2021-11-03 20:40:16 +01:00
urlEncodeListings = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2021-11-03 19:01:23 +01:00
case "IBMCOS" :
2021-11-03 20:40:16 +01:00
listObjectsV2 = false // untested
virtualHostStyle = false
urlEncodeListings = false
2022-02-27 16:47:31 +01:00
useMultipartEtag = false // untested
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // returns BucketAlreadyExists
2022-06-18 09:29:21 +02:00
case "IDrive" :
virtualHostStyle = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2022-09-15 17:04:34 +02:00
case "IONOS" :
// listObjectsV2 supported - https://api.ionos.com/docs/s3/#Basic-Operations-get-Bucket-list-type-2
virtualHostStyle = false
urlEncodeListings = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2023-05-05 10:44:25 +02:00
case "Petabox" :
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2022-12-06 13:25:23 +01:00
case "Liara" :
virtualHostStyle = false
urlEncodeListings = false
useMultipartEtag = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2023-09-21 13:38:06 +02:00
case "Linode" :
2023-10-06 12:45:03 +02:00
useAlreadyExists = true // returns 200 OK
2022-03-14 16:37:34 +01:00
case "LyveCloud" :
useMultipartEtag = false // LyveCloud seems to calculate multipart Etags differently from AWS
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2024-06-06 12:25:45 +02:00
case "Magalu" :
listObjectsV2 = false
virtualHostStyle = false
urlEncodeListings = false
useMultipartEtag = false
useAlreadyExists = false
2021-11-03 19:01:23 +01:00
case "Minio" :
2021-11-03 20:40:16 +01:00
virtualHostStyle = false
2021-11-03 19:01:23 +01:00
case "Netease" :
2021-11-03 20:40:16 +01:00
listObjectsV2 = false // untested
urlEncodeListings = false
2022-02-27 16:47:31 +01:00
useMultipartEtag = false // untested
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2021-11-09 12:46:58 +01:00
case "RackCorp" :
// No quirks
2022-02-27 16:47:31 +01:00
useMultipartEtag = false // untested
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2023-11-13 17:40:34 +01:00
case "Rclone" :
listObjectsV2 = true
urlEncodeListings = true
virtualHostStyle = false
useMultipartEtag = false
useAlreadyExists = false
// useMultipartUploads = false - set this manually
2024-08-06 12:50:21 +02:00
// rclone serve doesn't support multi-part server side copy:
// See: https://github.com/rclone/rclone/issues/7454
// So make cutoff very large which it does support
opt . CopyCutoff = math . MaxInt64
2021-11-03 19:01:23 +01:00
case "Scaleway" :
2021-11-03 20:40:16 +01:00
// Scaleway can only have 1000 parts in an upload
if opt . MaxUploadParts > 1000 {
opt . MaxUploadParts = 1000
}
2024-03-31 18:42:20 +02:00
urlEncodeListings = true
useAlreadyExists = true
2021-11-03 19:01:23 +01:00
case "SeaweedFS" :
2021-11-03 20:40:16 +01:00
listObjectsV2 = false // untested
virtualHostStyle = false
urlEncodeListings = false
2022-02-27 16:47:31 +01:00
useMultipartEtag = false // untested
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2021-11-03 19:01:23 +01:00
case "StackPath" :
2021-11-03 20:40:16 +01:00
listObjectsV2 = false // untested
virtualHostStyle = false
urlEncodeListings = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2021-09-29 15:43:06 +02:00
case "Storj" :
// Force chunk size to >= 64 MiB
if opt . ChunkSize < 64 * fs . Mebi {
opt . ChunkSize = 64 * fs . Mebi
}
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // returns BucketAlreadyExists
2023-07-06 11:54:07 +02:00
case "Synology" :
useMultipartEtag = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2021-11-03 19:01:23 +01:00
case "TencentCOS" :
2022-02-27 16:47:31 +01:00
listObjectsV2 = false // untested
useMultipartEtag = false // untested
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2021-11-03 19:01:23 +01:00
case "Wasabi" :
2023-10-06 12:45:03 +02:00
useAlreadyExists = true // returns 200 OK
2023-07-16 19:08:47 +02:00
case "Leviia" :
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2022-10-13 16:49:22 +02:00
case "Qiniu" :
useMultipartEtag = false
urlEncodeListings = false
2023-05-17 13:23:34 +02:00
virtualHostStyle = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false // untested
2023-01-09 07:54:51 +01:00
case "GCS" :
// Google break request Signature by mutating accept-encoding HTTP header
// https://github.com/rclone/rclone/issues/6670
useAcceptEncodingGzip = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = true // returns BucketNameUnavailable instead of BucketAlreadyExists but good enough!
2024-02-03 12:14:23 +01:00
// GCS S3 doesn't support multi-part server side copy:
// See: https://issuetracker.google.com/issues/323465186
// So make cutoff very large which it does seem to support
opt . CopyCutoff = math . MaxInt64
2024-06-01 11:43:49 +02:00
default : //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid defaultCaseOrder: consider to make `default` case as first or as last case
2021-11-03 20:40:16 +01:00
fs . Logf ( "s3" , "s3 provider %q not known - please set correctly" , opt . Provider )
2023-10-06 12:45:03 +02:00
fallthrough
case "Other" :
2021-11-03 20:40:16 +01:00
listObjectsV2 = false
virtualHostStyle = false
urlEncodeListings = false
2022-02-27 16:47:31 +01:00
useMultipartEtag = false
2023-10-06 12:45:03 +02:00
useAlreadyExists = false
2021-11-03 20:40:16 +01:00
}
// Path Style vs Virtual Host style
if virtualHostStyle || opt . UseAccelerateEndpoint {
opt . ForcePathStyle = false
}
// Set to see if we need to URL encode listings
if ! opt . ListURLEncode . Valid {
opt . ListURLEncode . Valid = true
opt . ListURLEncode . Value = urlEncodeListings
2021-11-03 19:01:23 +01:00
}
// Set the correct list version if not manually set
if opt . ListVersion == 0 {
2021-11-03 20:40:16 +01:00
if listObjectsV2 {
2021-11-03 19:01:23 +01:00
opt . ListVersion = 2
2021-11-03 20:40:16 +01:00
} else {
opt . ListVersion = 1
2021-11-03 19:01:23 +01:00
}
}
2022-02-27 16:47:31 +01:00
// Set the correct use multipart Etag for error checking if not manually set
if ! opt . UseMultipartEtag . Valid {
opt . UseMultipartEtag . Valid = true
opt . UseMultipartEtag . Value = useMultipartEtag
}
2022-10-29 00:23:29 +02:00
// set MightGzip if not manually set
if ! opt . MightGzip . Valid {
opt . MightGzip . Valid = true
opt . MightGzip . Value = mightGzip
}
2023-01-09 07:54:51 +01:00
// set UseAcceptEncodingGzip if not manually set
if ! opt . UseAcceptEncodingGzip . Valid {
opt . UseAcceptEncodingGzip . Valid = true
opt . UseAcceptEncodingGzip . Value = useAcceptEncodingGzip
}
2023-10-06 12:45:03 +02:00
// Has the provider got AlreadyOwnedByYou error?
if ! opt . UseAlreadyExists . Valid {
opt . UseAlreadyExists . Valid = true
opt . UseAlreadyExists . Value = useAlreadyExists
}
2023-11-14 13:39:50 +01:00
// Set the correct use multipart uploads if not manually set
if ! opt . UseMultipartUploads . Valid {
opt . UseMultipartUploads . Valid = true
opt . UseMultipartUploads . Value = useMultipartUploads
}
if ! opt . UseMultipartUploads . Value {
opt . UploadCutoff = math . MaxInt64
}
2024-08-03 12:35:32 +02:00
// Set the correct use multipart uploads if not manually set
if ! opt . UseUnsignedPayload . Valid {
opt . UseUnsignedPayload . Valid = true
opt . UseUnsignedPayload . Value = useUnsignedPayload
}
2021-11-03 19:01:23 +01:00
}
2019-08-09 12:29:36 +02:00
// setRoot changes the root of the Fs
func ( f * Fs ) setRoot ( root string ) {
f . root = parsePath ( root )
f . rootBucket , f . rootDirectory = bucket . Split ( f . root )
}
2016-09-01 23:27:50 +02:00
// NewFs constructs an Fs from the path, bucket:path
2020-11-05 16:18:51 +01:00
func NewFs ( ctx context . Context , name , root string , m configmap . Mapper ) ( fs . Fs , error ) {
2018-05-14 19:06:57 +02:00
// Parse config into Options struct
opt := new ( Options )
err := configstruct . Set ( m , opt )
if err != nil {
return nil , err
}
2018-09-07 13:02:27 +02:00
err = checkUploadChunkSize ( opt . ChunkSize )
if err != nil {
2021-11-04 11:12:57 +01:00
return nil , fmt . Errorf ( "s3: chunk size: %w" , err )
2018-05-14 19:06:57 +02:00
}
2018-11-26 22:09:23 +01:00
err = checkUploadCutoff ( opt . UploadCutoff )
if err != nil {
2021-11-04 11:12:57 +01:00
return nil , fmt . Errorf ( "s3: upload cutoff: %w" , err )
2018-11-26 22:09:23 +01:00
}
2024-03-13 12:53:38 +01:00
err = checkCopyCutoff ( opt . CopyCutoff )
if err != nil {
return nil , fmt . Errorf ( "s3: --s3-copy-cutoff: %w" , err )
}
2022-07-26 18:58:57 +02:00
if opt . Versions && opt . VersionAt . IsSet ( ) {
2022-09-20 08:54:47 +02:00
return nil , errors . New ( "s3: can't use --s3-versions and --s3-version-at at the same time" )
2022-07-26 18:58:57 +02:00
}
2019-01-16 18:23:37 +01:00
if opt . BucketACL == "" {
opt . BucketACL = opt . ACL
}
2022-09-17 18:28:44 +02:00
if opt . SSECustomerKeyBase64 != "" && opt . SSECustomerKey != "" {
return nil , errors . New ( "s3: can't use sse_customer_key and sse_customer_key_base64 at the same time" )
} else if opt . SSECustomerKeyBase64 != "" {
// Decode the base64-encoded key and store it in the SSECustomerKey field
decoded , err := base64 . StdEncoding . DecodeString ( opt . SSECustomerKeyBase64 )
if err != nil {
return nil , fmt . Errorf ( "s3: Could not decode sse_customer_key_base64: %w" , err )
}
opt . SSECustomerKey = string ( decoded )
2024-08-07 11:14:58 +02:00
} else {
// Encode the raw key as base64
opt . SSECustomerKeyBase64 = base64 . StdEncoding . EncodeToString ( [ ] byte ( opt . SSECustomerKey ) )
2022-09-17 18:28:44 +02:00
}
2020-11-20 12:15:48 +01:00
if opt . SSECustomerKey != "" && opt . SSECustomerKeyMD5 == "" {
// calculate CustomerKeyMD5 if not supplied
md5sumBinary := md5 . Sum ( [ ] byte ( opt . SSECustomerKey ) )
opt . SSECustomerKeyMD5 = base64 . StdEncoding . EncodeToString ( md5sumBinary [ : ] )
}
2021-03-05 12:04:57 +01:00
srv := getClient ( ctx , opt )
2024-08-03 12:35:32 +02:00
c , err := s3Connection ( ctx , opt , srv )
2013-01-08 19:53:35 +01:00
if err != nil {
return nil , err
}
2020-02-19 11:17:25 +01:00
2020-11-05 12:33:32 +01:00
ci := fs . GetConfig ( ctx )
2021-08-13 18:27:49 +02:00
pc := fs . NewPacer ( ctx , pacer . NewS3 ( pacer . MinSleep ( minSleep ) ) )
// Set pacer retries to 2 (1 try and 1 retry) because we are
// relying on SDK retry mechanism, but we allow 2 attempts to
// retry directory listings after XMLSyntaxError
pc . SetRetries ( 2 )
2015-11-07 12:14:46 +01:00
f := & Fs {
2021-10-14 12:19:38 +02:00
name : name ,
opt : * opt ,
ci : ci ,
ctx : ctx ,
c : c ,
2021-08-13 18:27:49 +02:00
pacer : pc ,
2021-10-14 12:19:38 +02:00
cache : bucket . NewCache ( ) ,
srv : srv ,
srvRest : rest . NewClient ( fshttp . NewClient ( ctx ) ) ,
2019-08-09 12:29:36 +02:00
}
2020-11-20 13:15:56 +01:00
if opt . ServerSideEncryption == "aws:kms" || opt . SSECustomerAlgorithm != "" {
// From: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
//
// Objects encrypted by SSE-S3 or plaintext have ETags that are an MD5
// digest of their data.
//
// Objects encrypted by SSE-C or SSE-KMS have ETags that are not an
// MD5 digest of their object data.
f . etagIsNotMD5 = true
}
2019-08-09 12:29:36 +02:00
f . setRoot ( root )
2017-08-09 16:27:43 +02:00
f . features = ( & fs . Features {
2019-08-09 12:29:36 +02:00
ReadMimeType : true ,
WriteMimeType : true ,
2022-05-24 13:32:39 +02:00
ReadMetadata : true ,
WriteMetadata : true ,
UserMetadata : true ,
2019-08-09 12:29:36 +02:00
BucketBased : true ,
BucketBasedRootOK : true ,
2019-09-09 21:44:50 +02:00
SetTier : true ,
GetTier : true ,
2020-06-19 11:28:34 +02:00
SlowModTime : true ,
2020-11-05 17:00:40 +01:00
} ) . Fill ( ctx , f )
2023-02-02 19:12:00 +01:00
if opt . Provider == "Storj" {
f . features . SetTier = false
f . features . GetTier = false
}
if opt . Provider == "IDrive" {
f . features . SetTier = false
}
2023-04-26 11:59:17 +02:00
if opt . DirectoryMarkers {
f . features . CanHaveEmptyDirectories = true
}
2023-02-02 19:12:00 +01:00
// f.listMultipartUploads()
2023-11-14 13:39:50 +01:00
if ! opt . UseMultipartUploads . Value {
fs . Debugf ( f , "Disabling multipart uploads" )
f . features . OpenChunkWriter = nil
}
2023-02-02 19:12:00 +01:00
2021-02-07 18:18:52 +01:00
if f . rootBucket != "" && f . rootDirectory != "" && ! opt . NoHeadObject && ! strings . HasSuffix ( root , "/" ) {
2020-11-20 12:15:48 +01:00
// Check to see if the (bucket,directory) is actually an existing file
oldRoot := f . root
newRoot , leaf := path . Split ( oldRoot )
f . setRoot ( newRoot )
_ , err := f . NewObject ( ctx , leaf )
if err != nil {
2021-02-08 14:22:12 +01:00
// File doesn't exist or is a directory so return old f
f . setRoot ( oldRoot )
return f , nil
2014-05-05 20:52:52 +02:00
}
2020-11-20 12:15:48 +01:00
// return an error with an fs which points to the parent
return f , fs . ErrorIsFile
2014-05-05 20:52:52 +02:00
}
2013-01-08 19:53:35 +01:00
return f , nil
}
2022-07-25 17:06:15 +02:00
// getMetaDataListing gets the metadata from the object unconditionally from the listing
//
// This is needed to find versioned objects from their paths.
//
// It may return info == nil and err == nil if a HEAD would be more appropriate
2024-08-03 12:35:32 +02:00
func ( f * Fs ) getMetaDataListing ( ctx context . Context , wantRemote string ) ( info * types . Object , versionID * string , err error ) {
2022-07-25 17:06:15 +02:00
bucket , bucketPath := f . split ( wantRemote )
2022-07-26 18:58:57 +02:00
// Strip the version string off if using versions
if f . opt . Versions {
var timestamp time . Time
timestamp , bucketPath = version . Remove ( bucketPath )
// If the path had no version string return no info, to force caller to look it up
if timestamp . IsZero ( ) {
return nil , nil , nil
}
2022-07-25 17:06:15 +02:00
}
2022-07-27 11:46:28 +02:00
err = f . list ( ctx , listOpt {
bucket : bucket ,
directory : bucketPath ,
2023-03-21 13:44:45 +01:00
prefix : f . rootDirectory ,
2022-07-27 11:46:28 +02:00
recurse : true ,
withVersions : f . opt . Versions ,
findFile : true ,
2022-07-26 18:58:57 +02:00
versionAt : f . opt . VersionAt ,
2023-11-08 16:29:23 +01:00
hidden : f . opt . VersionDeleted ,
2024-08-03 12:35:32 +02:00
} , func ( gotRemote string , object * types . Object , objectVersionID * string , isDirectory bool ) error {
2022-07-25 17:06:15 +02:00
if isDirectory {
return nil
}
if wantRemote != gotRemote {
return nil
}
info = object
versionID = objectVersionID
return errEndList // read only 1 item
} )
if err != nil {
if err == fs . ErrorDirNotFound {
return nil , nil , fs . ErrorObjectNotFound
}
return nil , nil , err
}
if info == nil {
return nil , nil , fs . ErrorObjectNotFound
}
return info , versionID , nil
}
2024-08-03 12:35:32 +02:00
// stringClone clones the string s into new memory. This is useful to
// stop us keeping references to small strings carved out of large XML
// responses.
func stringClone ( s string ) * string {
var sNew = strings . Clone ( s )
return & sNew
}
2022-12-08 13:41:23 +01:00
// stringClonePointer clones the string pointed to by sp into new
// memory. This is useful to stop us keeping references to small
// strings carved out of large XML responses.
func stringClonePointer ( sp * string ) * string {
if sp == nil {
return nil
}
2024-08-03 12:35:32 +02:00
var s = strings . Clone ( * sp )
2022-12-08 13:41:23 +01:00
return & s
}
2016-06-25 22:58:34 +02:00
// Return an Object from a path
2013-01-08 19:53:35 +01:00
//
2022-08-05 17:35:41 +02:00
// If it can't be found it returns the error ErrorObjectNotFound.
2024-08-03 12:35:32 +02:00
func ( f * Fs ) newObjectWithInfo ( ctx context . Context , remote string , info * types . Object , versionID * string ) ( obj fs . Object , err error ) {
2015-11-07 12:14:46 +01:00
o := & Object {
fs : f ,
2013-01-08 19:53:35 +01:00
remote : remote ,
}
2022-07-26 18:58:57 +02:00
if info == nil && ( ( f . opt . Versions && version . Match ( remote ) ) || f . opt . VersionAt . IsSet ( ) ) {
// If versions, have to read the listing to find the correct version ID
2022-07-25 17:06:15 +02:00
info , versionID , err = f . getMetaDataListing ( ctx , remote )
if err != nil {
return nil , err
}
}
2013-01-08 19:53:35 +01:00
if info != nil {
// Set info but not meta
2014-12-23 13:09:02 +01:00
if info . LastModified == nil {
2017-02-09 12:01:20 +01:00
fs . Logf ( o , "Failed to read last modified" )
2013-06-27 21:13:07 +02:00
o . lastModified = time . Now ( )
2014-12-23 13:09:02 +01:00
} else {
o . lastModified = * info . LastModified
2013-01-08 19:53:35 +01:00
}
2024-08-03 12:35:32 +02:00
o . setMD5FromEtag ( deref ( info . ETag ) )
o . bytes = deref ( info . Size )
o . storageClass = stringClone ( string ( info . StorageClass ) )
2022-12-13 12:50:51 +01:00
o . versionID = stringClonePointer ( versionID )
2023-11-08 16:29:23 +01:00
// If is delete marker, show that metadata has been read as there is none to read
if info . Size == isDeleteMarker {
o . meta = map [ string ] string { }
}
2021-04-28 12:05:54 +02:00
} else if ! o . fs . opt . NoHeadObject {
2019-06-17 10:34:30 +02:00
err := o . readMetaData ( ctx ) // reads info and meta, returning an error
2013-01-08 19:53:35 +01:00
if err != nil {
2016-06-25 22:23:20 +02:00
return nil , err
2013-01-08 19:53:35 +01:00
}
}
2016-06-25 22:23:20 +02:00
return o , nil
2013-01-08 19:53:35 +01:00
}
2016-06-25 22:23:20 +02:00
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
2019-06-17 10:34:30 +02:00
func ( f * Fs ) NewObject ( ctx context . Context , remote string ) ( fs . Object , error ) {
2022-07-25 17:06:15 +02:00
return f . newObjectWithInfo ( ctx , remote , nil , nil )
2013-01-08 19:53:35 +01:00
}
2019-01-16 14:35:19 +01:00
// Gets the bucket location
2021-03-16 16:50:02 +01:00
func ( f * Fs ) getBucketLocation ( ctx context . Context , bucket string ) ( string , error ) {
2024-08-03 12:35:32 +02:00
region , err := manager . GetBucketRegion ( ctx , f . c , bucket )
2019-01-16 14:35:19 +01:00
if err != nil {
return "" , err
}
2022-10-29 09:37:08 +02:00
return region , nil
2019-01-16 14:35:19 +01:00
}
// Updates the region for the bucket by reading the region from the
// bucket then updating the session.
2021-03-16 16:50:02 +01:00
func ( f * Fs ) updateRegionForBucket ( ctx context . Context , bucket string ) error {
region , err := f . getBucketLocation ( ctx , bucket )
2019-01-16 14:35:19 +01:00
if err != nil {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "reading bucket location failed: %w" , err )
2019-01-16 14:35:19 +01:00
}
2024-08-03 12:35:32 +02:00
if f . opt . Endpoint != "" {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "can't set region to %q as endpoint is set" , region )
2019-01-16 14:35:19 +01:00
}
2024-08-03 12:35:32 +02:00
if f . opt . Region == region {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "region is already %q - not updating" , region )
2019-01-16 14:35:19 +01:00
}
// Make a new session with the new region
oldRegion := f . opt . Region
f . opt . Region = region
2024-08-03 12:35:32 +02:00
c , err := s3Connection ( f . ctx , & f . opt , f . srv )
2019-01-16 14:35:19 +01:00
if err != nil {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "creating new session failed: %w" , err )
2019-01-16 14:35:19 +01:00
}
f . c = c
fs . Logf ( f , "Switched region to %q from %q" , region , oldRegion )
return nil
}
2022-07-25 16:56:17 +02:00
// Common interface for bucket listers
type bucketLister interface {
2022-07-26 18:58:57 +02:00
List ( ctx context . Context ) ( resp * s3 . ListObjectsV2Output , versionIDs [ ] * string , err error )
2022-07-25 16:56:17 +02:00
URLEncodeListings ( bool )
}
// V1 bucket lister
type v1List struct {
f * Fs
req s3 . ListObjectsInput
}
// Create a new V1 bucket lister
func ( f * Fs ) newV1List ( req * s3 . ListObjectsV2Input ) bucketLister {
l := & v1List {
f : f ,
}
// Convert v2 req into v1 req
2022-07-28 12:49:19 +02:00
//structs.SetFrom(&l.req, req)
setFrom_s3ListObjectsInput_s3ListObjectsV2Input ( & l . req , req )
2022-07-25 16:56:17 +02:00
return l
}
// List a bucket with V1 listing
2022-07-26 18:58:57 +02:00
func ( ls * v1List ) List ( ctx context . Context ) ( resp * s3 . ListObjectsV2Output , versionIDs [ ] * string , err error ) {
2024-08-03 12:35:32 +02:00
respv1 , err := ls . f . c . ListObjects ( ctx , & ls . req )
2022-07-25 16:56:17 +02:00
if err != nil {
return nil , nil , err
}
// Set up the request for next time
ls . req . Marker = respv1 . NextMarker
2024-08-03 12:35:32 +02:00
if deref ( respv1 . IsTruncated ) && ls . req . Marker == nil {
2022-07-25 16:56:17 +02:00
if len ( respv1 . Contents ) == 0 {
return nil , nil , errors . New ( "s3 protocol error: received listing v1 with IsTruncated set, no NextMarker and no Contents" )
}
// Use the last Key received if no NextMarker and isTruncated
ls . req . Marker = respv1 . Contents [ len ( respv1 . Contents ) - 1 ] . Key
}
// If we are URL encoding then must decode the marker
2024-08-03 12:35:32 +02:00
if ls . req . Marker != nil && ls . req . EncodingType == types . EncodingTypeUrl {
2022-07-25 16:56:17 +02:00
* ls . req . Marker , err = url . QueryUnescape ( * ls . req . Marker )
if err != nil {
return nil , nil , fmt . Errorf ( "failed to URL decode Marker %q: %w" , * ls . req . Marker , err )
}
}
// convert v1 resp into v2 resp
resp = new ( s3 . ListObjectsV2Output )
2022-07-28 12:49:19 +02:00
//structs.SetFrom(resp, respv1)
setFrom_s3ListObjectsV2Output_s3ListObjectsOutput ( resp , respv1 )
2022-07-25 16:56:17 +02:00
return resp , nil , nil
}
// URL Encode the listings
func ( ls * v1List ) URLEncodeListings ( encode bool ) {
if encode {
2024-08-03 12:35:32 +02:00
ls . req . EncodingType = types . EncodingTypeUrl
2022-07-25 16:56:17 +02:00
} else {
2024-08-03 12:35:32 +02:00
ls . req . EncodingType = types . EncodingType ( "" )
2022-07-25 16:56:17 +02:00
}
}
// V2 bucket lister
type v2List struct {
f * Fs
req s3 . ListObjectsV2Input
}
// Create a new V2 bucket lister
func ( f * Fs ) newV2List ( req * s3 . ListObjectsV2Input ) bucketLister {
return & v2List {
f : f ,
req : * req ,
}
}
// Do a V2 listing
2022-07-26 18:58:57 +02:00
func ( ls * v2List ) List ( ctx context . Context ) ( resp * s3 . ListObjectsV2Output , versionIDs [ ] * string , err error ) {
2024-08-03 12:35:32 +02:00
resp , err = ls . f . c . ListObjectsV2 ( ctx , & ls . req )
2022-11-10 12:52:59 +01:00
if err != nil {
return nil , nil , err
}
2024-08-03 12:35:32 +02:00
if deref ( resp . IsTruncated ) && ( resp . NextContinuationToken == nil || * resp . NextContinuationToken == "" ) {
2022-12-13 13:19:00 +01:00
return nil , nil , errors . New ( "s3 protocol error: received listing v2 with IsTruncated set and no NextContinuationToken. Should you be using `--s3-list-version 1`?" )
2022-12-05 17:59:48 +01:00
}
2022-07-25 16:56:17 +02:00
ls . req . ContinuationToken = resp . NextContinuationToken
2022-11-10 12:52:59 +01:00
return resp , nil , nil
2022-07-25 16:56:17 +02:00
}
// URL Encode the listings
func ( ls * v2List ) URLEncodeListings ( encode bool ) {
if encode {
2024-08-03 12:35:32 +02:00
ls . req . EncodingType = types . EncodingTypeUrl
2022-07-25 16:56:17 +02:00
} else {
2024-08-03 12:35:32 +02:00
ls . req . EncodingType = types . EncodingType ( "" )
2022-07-25 16:56:17 +02:00
}
}
2022-07-25 17:06:15 +02:00
// Versions bucket lister
type versionsList struct {
2022-07-26 18:58:57 +02:00
f * Fs
req s3 . ListObjectVersionsInput
versionAt time . Time // set if we want only versions before this
usingVersionAt bool // set if we need to use versionAt
hidden bool // set to see hidden versions
lastKeySent string // last Key sent to the receiving function
2022-07-25 17:06:15 +02:00
}
// Create a new Versions bucket lister
2022-07-26 18:58:57 +02:00
func ( f * Fs ) newVersionsList ( req * s3 . ListObjectsV2Input , hidden bool , versionAt time . Time ) bucketLister {
2022-07-25 17:06:15 +02:00
l := & versionsList {
2022-07-26 18:58:57 +02:00
f : f ,
versionAt : versionAt ,
usingVersionAt : ! versionAt . IsZero ( ) ,
hidden : hidden ,
2022-07-25 17:06:15 +02:00
}
// Convert v2 req into withVersions req
2022-07-28 12:49:19 +02:00
//structs.SetFrom(&l.req, req)
setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input ( & l . req , req )
2022-07-25 17:06:15 +02:00
return l
}
2024-08-03 12:35:32 +02:00
// Any types.Object or types.ObjectVersion with this as their Size are delete markers
2022-07-25 17:06:15 +02:00
var isDeleteMarker = new ( int64 )
2024-08-03 12:35:32 +02:00
// Compare two types.ObjectVersions, sorted alphabetically by key with
2022-07-26 18:58:57 +02:00
// the newest first if the Keys match or the one with IsLatest set if
// everything matches.
2024-08-03 12:35:32 +02:00
func versionLess ( a , b * types . ObjectVersion ) bool {
2022-07-26 18:58:57 +02:00
if a == nil || a . Key == nil || a . LastModified == nil {
return true
}
if b == nil || b . Key == nil || b . LastModified == nil {
return false
}
if * a . Key < * b . Key {
return true
}
if * a . Key > * b . Key {
return false
}
2024-05-31 15:14:40 +02:00
dt := a . LastModified . Sub ( * b . LastModified )
2022-07-26 18:58:57 +02:00
if dt > 0 {
return true
}
if dt < 0 {
return false
}
2024-08-03 12:35:32 +02:00
if deref ( a . IsLatest ) {
2022-07-26 18:58:57 +02:00
return true
}
return false
}
// Merge the DeleteMarkers into the Versions.
//
// These are delivered by S3 sorted by key then by LastUpdated
// newest first but annoyingly the SDK splits them up into two
// so we need to merge them back again
//
// We do this by converting the s3.DeleteEntry into
2024-08-03 12:35:32 +02:00
// types.ObjectVersion with Size = isDeleteMarker to tell them apart
2022-07-26 18:58:57 +02:00
//
// We then merge them back into the Versions in the correct order
2024-08-03 12:35:32 +02:00
func mergeDeleteMarkers ( oldVersions [ ] types . ObjectVersion , deleteMarkers [ ] types . DeleteMarkerEntry ) ( newVersions [ ] types . ObjectVersion ) {
newVersions = make ( [ ] types . ObjectVersion , 0 , len ( oldVersions ) + len ( deleteMarkers ) )
2022-07-26 18:58:57 +02:00
for _ , deleteMarker := range deleteMarkers {
2024-08-03 12:35:32 +02:00
var obj types . ObjectVersion
2022-07-28 12:49:19 +02:00
//structs.SetFrom(obj, deleteMarker)
2024-08-03 12:35:32 +02:00
setFrom_typesObjectVersion_typesDeleteMarkerEntry ( & obj , & deleteMarker )
2022-07-26 18:58:57 +02:00
obj . Size = isDeleteMarker
2024-08-03 12:35:32 +02:00
for len ( oldVersions ) > 0 && versionLess ( & oldVersions [ 0 ] , & obj ) {
2022-07-26 18:58:57 +02:00
newVersions = append ( newVersions , oldVersions [ 0 ] )
oldVersions = oldVersions [ 1 : ]
}
newVersions = append ( newVersions , obj )
}
// Merge any remaining versions
newVersions = append ( newVersions , oldVersions ... )
return newVersions
}
2022-07-25 17:06:15 +02:00
// List a bucket with versions
2022-07-26 18:58:57 +02:00
func ( ls * versionsList ) List ( ctx context . Context ) ( resp * s3 . ListObjectsV2Output , versionIDs [ ] * string , err error ) {
2024-08-03 12:35:32 +02:00
respVersions , err := ls . f . c . ListObjectVersions ( ctx , & ls . req )
2022-07-25 17:06:15 +02:00
if err != nil {
return nil , nil , err
}
// Set up the request for next time
ls . req . KeyMarker = respVersions . NextKeyMarker
ls . req . VersionIdMarker = respVersions . NextVersionIdMarker
2024-08-03 12:35:32 +02:00
if deref ( respVersions . IsTruncated ) && ls . req . KeyMarker == nil {
2023-11-21 11:30:47 +01:00
return nil , nil , errors . New ( "s3 protocol error: received versions listing with IsTruncated set with no NextKeyMarker" )
}
2022-07-25 17:06:15 +02:00
// If we are URL encoding then must decode the marker
2024-08-03 12:35:32 +02:00
if ls . req . KeyMarker != nil && ls . req . EncodingType == types . EncodingTypeUrl {
2022-07-25 17:06:15 +02:00
* ls . req . KeyMarker , err = url . QueryUnescape ( * ls . req . KeyMarker )
if err != nil {
return nil , nil , fmt . Errorf ( "failed to URL decode KeyMarker %q: %w" , * ls . req . KeyMarker , err )
}
}
// convert Versions resp into v2 resp
resp = new ( s3 . ListObjectsV2Output )
2022-07-28 12:49:19 +02:00
//structs.SetFrom(resp, respVersions)
setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput ( resp , respVersions )
2022-07-25 17:06:15 +02:00
2024-08-03 12:35:32 +02:00
// Merge in delete Markers as types.ObjectVersion if we need them
2022-07-26 18:58:57 +02:00
if ls . hidden || ls . usingVersionAt {
respVersions . Versions = mergeDeleteMarkers ( respVersions . Versions , respVersions . DeleteMarkers )
}
2024-08-03 12:35:32 +02:00
// Convert the Versions and the DeleteMarkers into an array of types.Object
2022-07-25 17:06:15 +02:00
//
// These are returned in the order that they are stored with the most recent first.
// With the annoyance that the Versions and DeleteMarkers are split into two
2024-08-03 12:35:32 +02:00
objs := make ( [ ] types . Object , 0 , len ( respVersions . Versions ) )
2022-07-25 17:06:15 +02:00
for _ , objVersion := range respVersions . Versions {
2022-07-26 18:58:57 +02:00
if ls . usingVersionAt {
if objVersion . LastModified . After ( ls . versionAt ) {
// Ignore versions that were created after the specified time
continue
}
if * objVersion . Key == ls . lastKeySent {
// Ignore versions before the already returned version
continue
}
}
ls . lastKeySent = * objVersion . Key
// Don't send delete markers if we don't want hidden things
if ! ls . hidden && objVersion . Size == isDeleteMarker {
continue
}
2024-08-03 12:35:32 +02:00
var obj types . Object
2022-07-28 12:49:19 +02:00
//structs.SetFrom(obj, objVersion)
2024-08-03 12:35:32 +02:00
setFrom_typesObject_typesObjectVersion ( & obj , & objVersion )
2022-07-25 17:06:15 +02:00
// Adjust the file names
2024-08-03 12:35:32 +02:00
if ! ls . usingVersionAt && ( ! deref ( objVersion . IsLatest ) || objVersion . Size == isDeleteMarker ) {
2022-07-25 17:06:15 +02:00
if obj . Key != nil && objVersion . LastModified != nil {
* obj . Key = version . Add ( * obj . Key , * objVersion . LastModified )
}
}
objs = append ( objs , obj )
versionIDs = append ( versionIDs , objVersion . VersionId )
}
resp . Contents = objs
return resp , versionIDs , nil
}
// URL Encode the listings
func ( ls * versionsList ) URLEncodeListings ( encode bool ) {
if encode {
2024-08-03 12:35:32 +02:00
ls . req . EncodingType = types . EncodingTypeUrl
2022-07-25 17:06:15 +02:00
} else {
2024-08-03 12:35:32 +02:00
ls . req . EncodingType = types . EncodingType ( "" )
2022-07-25 17:06:15 +02:00
}
}
2016-04-21 21:06:21 +02:00
// listFn is called from list to handle an object.
2024-08-03 12:35:32 +02:00
type listFn func ( remote string , object * types . Object , versionID * string , isDirectory bool ) error
2022-07-25 17:06:15 +02:00
// errEndList is a sentinel used to end the list iteration now.
// listFn should return it to end the iteration with no errors.
var errEndList = errors . New ( "end list" )
2016-04-21 21:06:21 +02:00
2022-07-27 11:46:28 +02:00
// list options
type listOpt struct {
2023-02-27 16:32:59 +01:00
bucket string // bucket to list
directory string // directory with bucket
prefix string // prefix to remove from listing
addBucket bool // if set, the bucket is added to the start of the remote
recurse bool // if set, recurse to read sub directories
withVersions bool // if set, versions are produced
hidden bool // if set, return delete markers as objects with size == isDeleteMarker
findFile bool // if set, it will look for files called (bucket, directory)
versionAt fs . Time // if set only show versions <= this time
noSkipMarkers bool // if set return dir marker objects
2023-09-06 13:42:03 +02:00
restoreStatus bool // if set return restore status in listing too
2022-07-27 11:46:28 +02:00
}
// list lists the objects into the function supplied with the opt
// supplied.
func ( f * Fs ) list ( ctx context . Context , opt listOpt , fn listFn ) error {
2023-03-21 13:44:45 +01:00
if opt . prefix != "" {
opt . prefix += "/"
}
2022-07-27 11:46:28 +02:00
if ! opt . findFile {
if opt . directory != "" {
opt . directory += "/"
2022-07-25 17:06:15 +02:00
}
2016-04-23 22:46:52 +02:00
}
2014-05-05 19:25:32 +02:00
delimiter := ""
2022-07-27 11:46:28 +02:00
if ! opt . recurse {
2014-05-05 19:25:32 +02:00
delimiter = "/"
}
2019-09-16 21:25:55 +02:00
// URL encode the listings so we can use control characters in object names
// See: https://github.com/aws/aws-sdk-go/issues/1914
//
// However this doesn't work perfectly under Ceph (and hence DigitalOcean/Dreamhost) because
// it doesn't encode CommonPrefixes.
// See: https://tracker.ceph.com/issues/41870
//
// This does not work under IBM COS also: See https://github.com/rclone/rclone/issues/3345
// though maybe it does on some versions.
//
// This does work with minio but was only added relatively recently
// https://github.com/minio/minio/pull/7265
//
// So we enable only on providers we know supports it properly, all others can retry when a
// XML Syntax error is detected.
2021-11-03 20:40:16 +01:00
urlEncodeListings := f . opt . ListURLEncode . Value
2022-07-25 16:56:17 +02:00
req := s3 . ListObjectsV2Input {
2022-07-27 11:46:28 +02:00
Bucket : & opt . bucket ,
2022-07-25 16:56:17 +02:00
Delimiter : & delimiter ,
2022-07-27 11:46:28 +02:00
Prefix : & opt . directory ,
2022-07-25 16:56:17 +02:00
MaxKeys : & f . opt . ListChunk ,
}
2023-09-06 13:42:03 +02:00
if opt . restoreStatus {
2024-08-03 12:35:32 +02:00
req . OptionalObjectAttributes = [ ] types . OptionalObjectAttributes { types . OptionalObjectAttributesRestoreStatus }
2023-09-06 13:42:03 +02:00
}
2022-07-25 16:56:17 +02:00
if f . opt . RequesterPays {
2024-08-03 12:35:32 +02:00
req . RequestPayer = types . RequestPayerRequester
2022-07-25 16:56:17 +02:00
}
var listBucket bucketLister
switch {
2022-07-26 18:58:57 +02:00
case opt . withVersions || opt . versionAt . IsSet ( ) :
listBucket = f . newVersionsList ( & req , opt . hidden , time . Time ( opt . versionAt ) )
2022-07-25 16:56:17 +02:00
case f . opt . ListVersion == 1 :
listBucket = f . newV1List ( & req )
default :
listBucket = f . newV2List ( & req )
}
2023-04-26 11:59:17 +02:00
foundItems := 0
2015-02-10 18:58:29 +01:00
for {
2021-03-02 23:36:50 +01:00
var resp * s3 . ListObjectsV2Output
2018-09-03 06:41:04 +02:00
var err error
2022-07-25 17:06:15 +02:00
var versionIDs [ ] * string
2018-09-03 06:41:04 +02:00
err = f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
2022-07-25 16:56:17 +02:00
listBucket . URLEncodeListings ( urlEncodeListings )
2022-07-26 18:58:57 +02:00
resp , versionIDs , err = listBucket . List ( ctx )
2019-09-16 21:25:55 +02:00
if err != nil && ! urlEncodeListings {
2024-08-03 12:35:32 +02:00
var xmlErr * xml . SyntaxError
if errors . As ( err , & xmlErr ) {
// Retry the listing with URL encoding as there were characters that XML can't encode
urlEncodeListings = true
fs . Debugf ( f , "Retrying listing because of characters which can't be XML encoded" )
return true , err
2019-09-16 21:25:55 +02:00
}
}
2021-03-16 16:50:02 +01:00
return f . shouldRetry ( ctx , err )
2018-09-03 06:41:04 +02:00
} )
2015-02-10 18:58:29 +01:00
if err != nil {
2024-08-03 12:35:32 +02:00
if getHTTPStatusCode ( err ) == http . StatusNotFound {
err = fs . ErrorDirNotFound
2017-06-11 23:43:31 +02:00
}
2019-08-09 12:29:36 +02:00
if f . rootBucket == "" {
// if listing from the root ignore wrong region requests returning
// empty directory
2024-08-03 12:35:32 +02:00
// 301 if wrong region for bucket
if getHTTPStatusCode ( err ) == http . StatusMovedPermanently {
fs . Errorf ( f , "Can't change region for bucket %q with no bucket specified" , opt . bucket )
return nil
2019-08-09 12:29:36 +02:00
}
}
2016-04-21 21:06:21 +02:00
return err
}
2022-07-27 11:46:28 +02:00
if ! opt . recurse {
2023-04-26 11:59:17 +02:00
foundItems += len ( resp . CommonPrefixes )
2016-04-21 21:06:21 +02:00
for _ , commonPrefix := range resp . CommonPrefixes {
if commonPrefix . Prefix == nil {
2017-02-09 12:01:20 +01:00
fs . Logf ( f , "Nil common prefix received" )
2016-04-21 21:06:21 +02:00
continue
2015-02-10 18:58:29 +01:00
}
2016-04-21 21:06:21 +02:00
remote := * commonPrefix . Prefix
2019-09-16 21:25:55 +02:00
if urlEncodeListings {
remote , err = url . QueryUnescape ( remote )
if err != nil {
fs . Logf ( f , "failed to URL decode %q in listing common prefix: %v" , * commonPrefix . Prefix , err )
continue
}
2019-07-23 13:24:10 +02:00
}
2020-01-14 18:33:35 +01:00
remote = f . opt . Enc . ToStandardPath ( remote )
2022-07-27 11:46:28 +02:00
if ! strings . HasPrefix ( remote , opt . prefix ) {
2017-02-09 12:01:20 +01:00
fs . Logf ( f , "Odd name received %q" , remote )
2016-04-21 21:06:21 +02:00
continue
}
2022-07-27 11:46:28 +02:00
remote = remote [ len ( opt . prefix ) : ]
if opt . addBucket {
2023-03-03 11:45:29 +01:00
remote = bucket . Join ( opt . bucket , remote )
2019-08-09 12:29:36 +02:00
}
2022-06-08 22:25:17 +02:00
remote = strings . TrimSuffix ( remote , "/" )
2024-08-03 12:35:32 +02:00
err = fn ( remote , & types . Object { Key : & remote } , nil , true )
2016-04-21 21:06:21 +02:00
if err != nil {
2022-07-25 17:06:15 +02:00
if err == errEndList {
return nil
}
2016-04-21 21:06:21 +02:00
return err
2013-01-08 19:53:35 +01:00
}
}
2016-04-21 21:06:21 +02:00
}
2023-04-26 11:59:17 +02:00
foundItems += len ( resp . Contents )
2022-07-25 17:06:15 +02:00
for i , object := range resp . Contents {
2024-08-03 12:35:32 +02:00
remote := deref ( object . Key )
2019-09-16 21:25:55 +02:00
if urlEncodeListings {
remote , err = url . QueryUnescape ( remote )
if err != nil {
2024-08-03 12:35:32 +02:00
fs . Logf ( f , "failed to URL decode %q in listing: %v" , deref ( object . Key ) , err )
2019-09-16 21:25:55 +02:00
continue
}
2019-07-23 13:24:10 +02:00
}
2020-01-14 18:33:35 +01:00
remote = f . opt . Enc . ToStandardPath ( remote )
2022-07-27 11:46:28 +02:00
if ! strings . HasPrefix ( remote , opt . prefix ) {
2019-08-09 12:29:36 +02:00
fs . Logf ( f , "Odd name received %q" , remote )
2016-04-21 21:06:21 +02:00
continue
2014-12-23 13:09:02 +01:00
}
2023-04-26 11:59:17 +02:00
isDirectory := ( remote == "" || strings . HasSuffix ( remote , "/" ) ) && object . Size != nil && * object . Size == 0
2018-03-19 18:41:46 +01:00
// is this a directory marker?
2023-04-26 11:59:17 +02:00
if isDirectory {
if opt . noSkipMarkers {
// process directory markers as files
isDirectory = false
2024-05-31 15:18:56 +02:00
} else if remote == f . opt . Enc . ToStandardPath ( opt . directory ) {
2023-04-26 11:59:17 +02:00
// Don't insert the root directory
2024-05-31 15:18:56 +02:00
continue
2023-04-26 11:59:17 +02:00
}
2018-03-19 18:41:46 +01:00
}
2023-06-10 15:18:59 +02:00
remote = remote [ len ( opt . prefix ) : ]
2023-09-24 17:51:58 +02:00
if isDirectory {
// process directory markers as directories
remote = strings . TrimRight ( remote , "/" )
}
2023-06-10 15:18:59 +02:00
if opt . addBucket {
remote = bucket . Join ( opt . bucket , remote )
}
2022-07-25 17:06:15 +02:00
if versionIDs != nil {
2024-08-03 12:35:32 +02:00
err = fn ( remote , & object , versionIDs [ i ] , isDirectory )
2022-07-25 17:06:15 +02:00
} else {
2024-08-03 12:35:32 +02:00
err = fn ( remote , & object , nil , isDirectory )
2022-07-25 17:06:15 +02:00
}
2016-04-21 21:06:21 +02:00
if err != nil {
2022-07-25 17:06:15 +02:00
if err == errEndList {
return nil
}
2016-04-21 21:06:21 +02:00
return err
2014-12-23 13:09:02 +01:00
}
2015-02-10 18:58:29 +01:00
}
2024-08-03 12:35:32 +02:00
if ! deref ( resp . IsTruncated ) {
2016-04-21 21:06:21 +02:00
break
}
2014-05-05 19:25:32 +02:00
}
2023-04-26 11:59:17 +02:00
if f . opt . DirectoryMarkers && foundItems == 0 && opt . directory != "" {
// Determine whether the directory exists or not by whether it has a marker
req := s3 . HeadObjectInput {
Bucket : & opt . bucket ,
Key : & opt . directory ,
}
_ , err := f . headObject ( ctx , & req )
if err != nil {
if err == fs . ErrorObjectNotFound {
return fs . ErrorDirNotFound
}
return err
}
}
2016-04-21 21:06:21 +02:00
return nil
2014-05-05 19:25:32 +02:00
}
2017-06-30 11:54:14 +02:00
// Convert a list item into a DirEntry
2024-08-03 12:35:32 +02:00
func ( f * Fs ) itemToDirEntry ( ctx context . Context , remote string , object * types . Object , versionID * string , isDirectory bool ) ( fs . DirEntry , error ) {
2017-06-11 23:43:31 +02:00
if isDirectory {
size := int64 ( 0 )
if object . Size != nil {
size = * object . Size
}
2017-06-30 14:37:29 +02:00
d := fs . NewDir ( remote , time . Time { } ) . SetSize ( size )
2017-06-11 23:43:31 +02:00
return d , nil
2016-04-21 21:06:21 +02:00
}
2022-07-25 17:06:15 +02:00
o , err := f . newObjectWithInfo ( ctx , remote , object , versionID )
2017-06-11 23:43:31 +02:00
if err != nil {
return nil , err
}
return o , nil
}
// listDir lists files and directories to out
2019-08-09 12:29:36 +02:00
func ( f * Fs ) listDir ( ctx context . Context , bucket , directory , prefix string , addBucket bool ) ( entries fs . DirEntries , err error ) {
2016-04-21 21:06:21 +02:00
// List the objects and directories
2022-07-27 11:46:28 +02:00
err = f . list ( ctx , listOpt {
bucket : bucket ,
directory : directory ,
prefix : prefix ,
addBucket : addBucket ,
withVersions : f . opt . Versions ,
2022-07-26 18:58:57 +02:00
versionAt : f . opt . VersionAt ,
2023-11-08 16:29:23 +01:00
hidden : f . opt . VersionDeleted ,
2024-08-03 12:35:32 +02:00
} , func ( remote string , object * types . Object , versionID * string , isDirectory bool ) error {
2022-07-25 17:06:15 +02:00
entry , err := f . itemToDirEntry ( ctx , remote , object , versionID , isDirectory )
2017-06-11 23:43:31 +02:00
if err != nil {
return err
}
if entry != nil {
entries = append ( entries , entry )
2016-04-21 21:06:21 +02:00
}
return nil
} )
if err != nil {
2017-06-11 23:43:31 +02:00
return nil , err
2016-04-21 21:06:21 +02:00
}
2018-03-01 13:11:34 +01:00
// bucket must be present if listing succeeded
2019-08-09 12:29:36 +02:00
f . cache . MarkOK ( bucket )
2017-06-11 23:43:31 +02:00
return entries , nil
2016-04-21 21:06:21 +02:00
}
// listBuckets lists the buckets to out
2019-08-22 22:30:55 +02:00
func ( f * Fs ) listBuckets ( ctx context . Context ) ( entries fs . DirEntries , err error ) {
2016-04-21 21:06:21 +02:00
req := s3 . ListBucketsInput { }
2018-09-03 06:41:04 +02:00
var resp * s3 . ListBucketsOutput
err = f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
resp , err = f . c . ListBuckets ( ctx , & req )
2021-03-16 16:50:02 +01:00
return f . shouldRetry ( ctx , err )
2018-09-03 06:41:04 +02:00
} )
2016-04-21 21:06:21 +02:00
if err != nil {
2017-06-11 23:43:31 +02:00
return nil , err
2016-04-21 21:06:21 +02:00
}
for _ , bucket := range resp . Buckets {
2024-08-03 12:35:32 +02:00
bucketName := f . opt . Enc . ToStandardName ( deref ( bucket . Name ) )
2019-08-09 12:29:36 +02:00
f . cache . MarkOK ( bucketName )
2024-08-03 12:35:32 +02:00
d := fs . NewDir ( bucketName , deref ( bucket . CreationDate ) )
2017-06-11 23:43:31 +02:00
entries = append ( entries , d )
2014-05-05 19:25:32 +02:00
}
2017-06-11 23:43:31 +02:00
return entries , nil
2013-01-08 19:53:35 +01:00
}
2017-06-11 23:43:31 +02:00
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
2019-06-17 10:34:30 +02:00
func ( f * Fs ) List ( ctx context . Context , dir string ) ( entries fs . DirEntries , err error ) {
2019-08-09 12:29:36 +02:00
bucket , directory := f . split ( dir )
if bucket == "" {
2019-08-22 22:30:55 +02:00
if directory != "" {
return nil , fs . ErrorListBucketRequired
}
return f . listBuckets ( ctx )
2014-05-05 19:25:32 +02:00
}
2019-08-09 12:29:36 +02:00
return f . listDir ( ctx , bucket , directory , f . rootDirectory , f . rootBucket == "" )
2013-01-23 23:43:20 +01:00
}
2017-06-05 17:14:24 +02:00
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
2017-06-11 23:43:31 +02:00
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
2019-08-09 12:29:36 +02:00
// of listing recursively than doing a directory traversal.
2019-06-17 10:34:30 +02:00
func ( f * Fs ) ListR ( ctx context . Context , dir string , callback fs . ListRCallback ) ( err error ) {
2019-08-09 12:29:36 +02:00
bucket , directory := f . split ( dir )
2018-01-12 17:30:54 +01:00
list := walk . NewListRHelper ( callback )
2019-08-09 12:29:36 +02:00
listR := func ( bucket , directory , prefix string , addBucket bool ) error {
2022-07-27 11:46:28 +02:00
return f . list ( ctx , listOpt {
bucket : bucket ,
directory : directory ,
prefix : prefix ,
addBucket : addBucket ,
recurse : true ,
withVersions : f . opt . Versions ,
2022-07-26 18:58:57 +02:00
versionAt : f . opt . VersionAt ,
2023-11-08 16:29:23 +01:00
hidden : f . opt . VersionDeleted ,
2024-08-03 12:35:32 +02:00
} , func ( remote string , object * types . Object , versionID * string , isDirectory bool ) error {
2022-07-25 17:06:15 +02:00
entry , err := f . itemToDirEntry ( ctx , remote , object , versionID , isDirectory )
2019-08-09 12:29:36 +02:00
if err != nil {
return err
}
return list . Add ( entry )
} )
}
if bucket == "" {
2019-08-22 22:30:55 +02:00
entries , err := f . listBuckets ( ctx )
2019-08-09 12:29:36 +02:00
if err != nil {
return err
}
for _ , entry := range entries {
err = list . Add ( entry )
if err != nil {
return err
}
bucket := entry . Remote ( )
err = listR ( bucket , "" , f . rootDirectory , true )
if err != nil {
return err
}
2019-08-22 22:30:55 +02:00
// bucket must be present if listing succeeded
f . cache . MarkOK ( bucket )
2019-08-09 12:29:36 +02:00
}
} else {
err = listR ( bucket , directory , f . rootDirectory , f . rootBucket == "" )
2017-06-11 23:43:31 +02:00
if err != nil {
return err
}
2019-08-22 22:30:55 +02:00
// bucket must be present if listing succeeded
f . cache . MarkOK ( bucket )
2017-06-11 23:43:31 +02:00
}
return list . Flush ( )
2017-06-05 17:14:24 +02:00
}
2016-06-25 22:58:34 +02:00
// Put the Object into the bucket
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Put ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( fs . Object , error ) {
2015-11-07 12:14:46 +01:00
// Temporary Object under construction
fs := & Object {
fs : f ,
2016-02-18 12:35:25 +01:00
remote : src . Remote ( ) ,
2015-11-07 12:14:46 +01:00
}
2019-06-17 10:34:30 +02:00
return fs , fs . Update ( ctx , in , src , options ... )
2013-01-08 19:53:35 +01:00
}
2017-09-15 20:20:32 +02:00
// PutStream uploads to the remote path with the modTime given of indeterminate size
2019-06-17 10:34:30 +02:00
func ( f * Fs ) PutStream ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( fs . Object , error ) {
return f . Put ( ctx , in , src , options ... )
2017-09-15 20:20:32 +02:00
}
2016-02-24 02:58:55 +01:00
// Check if the bucket exists
2017-06-29 13:26:14 +02:00
//
// NB this can return incorrect results if called immediately after bucket deletion
2019-08-09 12:29:36 +02:00
func ( f * Fs ) bucketExists ( ctx context . Context , bucket string ) ( bool , error ) {
2016-02-24 02:58:55 +01:00
req := s3 . HeadBucketInput {
2019-08-09 12:29:36 +02:00
Bucket : & bucket ,
2016-02-24 02:58:55 +01:00
}
2018-09-03 06:41:04 +02:00
err := f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
_ , err := f . c . HeadBucket ( ctx , & req )
2021-03-16 16:50:02 +01:00
return f . shouldRetry ( ctx , err )
2018-09-03 06:41:04 +02:00
} )
2016-02-24 02:58:55 +01:00
if err == nil {
return true , nil
}
2024-08-03 12:35:32 +02:00
if getHTTPStatusCode ( err ) == http . StatusNotFound {
return false , nil
2016-02-24 02:58:55 +01:00
}
return false , err
}
2023-04-26 11:59:17 +02:00
// Create directory marker file and parents
func ( f * Fs ) createDirectoryMarker ( ctx context . Context , bucket , dir string ) error {
if ! f . opt . DirectoryMarkers || bucket == "" {
return nil
}
// Object to be uploaded
o := & Object {
fs : f ,
meta : map [ string ] string {
metaMtime : swift . TimeToFloatString ( time . Now ( ) ) ,
} ,
}
for {
_ , bucketPath := f . split ( dir )
// Don't create the directory marker if it is the bucket or at the very root
if bucketPath == "" {
break
}
o . remote = dir + "/"
// Check to see if object already exists
_ , err := o . headObject ( ctx )
if err == nil {
return nil
}
// Upload it if not
fs . Debugf ( o , "Creating directory marker" )
content := io . Reader ( strings . NewReader ( "" ) )
2023-04-28 18:22:45 +02:00
err = o . Update ( ctx , content , o )
2023-04-26 11:59:17 +02:00
if err != nil {
return fmt . Errorf ( "creating directory marker failed: %w" , err )
}
// Now check parent directory exists
dir = path . Dir ( dir )
if dir == "/" || dir == "." {
break
}
}
return nil
}
2013-01-08 19:53:35 +01:00
// Mkdir creates the bucket if it doesn't exist
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Mkdir ( ctx context . Context , dir string ) error {
2019-08-09 12:29:36 +02:00
bucket , _ := f . split ( dir )
2022-09-26 08:59:57 +02:00
e := f . makeBucket ( ctx , bucket )
if e != nil {
return e
}
2023-04-26 11:59:17 +02:00
return f . createDirectoryMarker ( ctx , bucket , dir )
}
// mkdirParent creates the parent bucket/directory if it doesn't exist
func ( f * Fs ) mkdirParent ( ctx context . Context , remote string ) error {
remote = strings . TrimRight ( remote , "/" )
dir := path . Dir ( remote )
if dir == "/" || dir == "." {
dir = ""
2022-09-26 08:59:57 +02:00
}
2023-04-26 11:59:17 +02:00
return f . Mkdir ( ctx , dir )
2019-08-22 22:30:55 +02:00
}
// makeBucket creates the bucket if it doesn't exist
func ( f * Fs ) makeBucket ( ctx context . Context , bucket string ) error {
2020-07-22 13:02:17 +02:00
if f . opt . NoCheckBucket {
return nil
}
2019-08-09 12:29:36 +02:00
return f . cache . Create ( bucket , func ( ) error {
req := s3 . CreateBucketInput {
Bucket : & bucket ,
2024-08-03 12:35:32 +02:00
ACL : types . BucketCannedACL ( f . opt . BucketACL ) ,
2017-06-29 13:26:14 +02:00
}
2019-08-09 12:29:36 +02:00
if f . opt . LocationConstraint != "" {
2024-08-03 12:35:32 +02:00
req . CreateBucketConfiguration = & types . CreateBucketConfiguration {
LocationConstraint : types . BucketLocationConstraint ( f . opt . LocationConstraint ) ,
2019-08-09 12:29:36 +02:00
}
2017-06-29 13:26:14 +02:00
}
2019-08-09 12:29:36 +02:00
err := f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
_ , err := f . c . CreateBucket ( ctx , & req )
2021-03-16 16:50:02 +01:00
return f . shouldRetry ( ctx , err )
2019-08-09 12:29:36 +02:00
} )
if err == nil {
fs . Infof ( f , "Bucket %q created with ACL %q" , bucket , f . opt . BucketACL )
2014-12-23 13:09:02 +01:00
}
2024-08-03 12:35:32 +02:00
var awsErr smithy . APIError
if errors . As ( err , & awsErr ) {
switch awsErr . ErrorCode ( ) {
2023-10-06 12:45:03 +02:00
case "BucketAlreadyOwnedByYou" :
2019-08-09 12:29:36 +02:00
err = nil
2023-10-06 12:45:03 +02:00
case "BucketAlreadyExists" , "BucketNameUnavailable" :
if f . opt . UseAlreadyExists . Value {
// We can trust BucketAlreadyExists to mean not owned by us, so make it non retriable
err = fserrors . NoRetryError ( err )
} else {
// We can't trust BucketAlreadyExists to mean not owned by us, so ignore it
err = nil
}
2019-08-09 12:29:36 +02:00
}
2013-01-08 23:31:16 +01:00
}
2020-04-15 14:13:13 +02:00
return err
2019-08-09 12:29:36 +02:00
} , func ( ) ( bool , error ) {
return f . bucketExists ( ctx , bucket )
} )
2013-01-08 19:53:35 +01:00
}
2015-11-07 16:31:04 +01:00
// Rmdir deletes the bucket if the fs is at the root
2013-01-08 19:53:35 +01:00
//
// Returns an error if it isn't empty
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Rmdir ( ctx context . Context , dir string ) error {
2019-08-09 12:29:36 +02:00
bucket , directory := f . split ( dir )
2022-09-26 08:59:57 +02:00
// Remove directory marker file
if f . opt . DirectoryMarkers && bucket != "" && dir != "" {
2023-04-26 11:59:17 +02:00
o := & Object {
2022-09-26 08:59:57 +02:00
fs : f ,
2023-04-26 11:59:17 +02:00
remote : dir + "/" ,
}
fs . Debugf ( o , "Removing directory marker" )
err := o . Remove ( ctx )
if err != nil {
return fmt . Errorf ( "removing directory marker failed: %w" , err )
2022-09-26 08:59:57 +02:00
}
}
2019-08-09 12:29:36 +02:00
if bucket == "" || directory != "" {
2015-11-07 16:31:04 +01:00
return nil
}
2019-08-09 12:29:36 +02:00
return f . cache . Remove ( bucket , func ( ) error {
req := s3 . DeleteBucketInput {
Bucket : & bucket ,
}
err := f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
_ , err := f . c . DeleteBucket ( ctx , & req )
2021-03-16 16:50:02 +01:00
return f . shouldRetry ( ctx , err )
2019-08-09 12:29:36 +02:00
} )
if err == nil {
fs . Infof ( f , "Bucket %q deleted" , bucket )
}
return err
2018-09-03 06:41:04 +02:00
} )
2013-01-08 19:53:35 +01:00
}
2015-09-22 19:47:16 +02:00
// Precision of the remote
2015-11-07 12:14:46 +01:00
func ( f * Fs ) Precision ( ) time . Duration {
2013-01-19 00:21:02 +01:00
return time . Nanosecond
}
2018-01-23 11:50:50 +01:00
// pathEscape escapes s as for a URL path. It uses rest.URLPathEscape
// but also escapes '+' for S3 and Digital Ocean spaces compatibility
func pathEscape ( s string ) string {
2022-05-16 18:11:45 +02:00
return strings . ReplaceAll ( rest . URLPathEscape ( s ) , "+" , "%2B" )
2018-01-23 11:50:50 +01:00
}
2020-10-13 23:43:40 +02:00
// copy does a server-side copy
2019-09-09 21:44:50 +02:00
//
// It adds the boiler plate to the req passed in and calls the s3
// method
2020-07-30 11:52:32 +02:00
func ( f * Fs ) copy ( ctx context . Context , req * s3 . CopyObjectInput , dstBucket , dstPath , srcBucket , srcPath string , src * Object ) error {
2019-09-09 21:44:50 +02:00
req . Bucket = & dstBucket
2024-08-03 12:35:32 +02:00
req . ACL = types . ObjectCannedACL ( f . opt . ACL )
2019-09-09 21:44:50 +02:00
req . Key = & dstPath
2023-03-03 11:45:29 +01:00
source := pathEscape ( bucket . Join ( srcBucket , srcPath ) )
2022-07-25 17:06:15 +02:00
if src . versionID != nil {
source += fmt . Sprintf ( "?versionId=%s" , * src . versionID )
}
2019-09-09 21:44:50 +02:00
req . CopySource = & source
2020-12-03 03:30:06 +01:00
if f . opt . RequesterPays {
2024-08-03 12:35:32 +02:00
req . RequestPayer = types . RequestPayerRequester
2020-12-03 03:30:06 +01:00
}
2019-09-09 21:44:50 +02:00
if f . opt . ServerSideEncryption != "" {
2024-08-03 12:35:32 +02:00
req . ServerSideEncryption = types . ServerSideEncryption ( f . opt . ServerSideEncryption )
2019-09-09 21:44:50 +02:00
}
2020-11-20 12:15:48 +01:00
if f . opt . SSECustomerAlgorithm != "" {
req . SSECustomerAlgorithm = & f . opt . SSECustomerAlgorithm
req . CopySourceSSECustomerAlgorithm = & f . opt . SSECustomerAlgorithm
}
2024-08-07 11:14:58 +02:00
if f . opt . SSECustomerKeyBase64 != "" {
req . SSECustomerKey = & f . opt . SSECustomerKeyBase64
req . CopySourceSSECustomerKey = & f . opt . SSECustomerKeyBase64
2020-11-20 12:15:48 +01:00
}
if f . opt . SSECustomerKeyMD5 != "" {
req . SSECustomerKeyMD5 = & f . opt . SSECustomerKeyMD5
req . CopySourceSSECustomerKeyMD5 = & f . opt . SSECustomerKeyMD5
}
2019-09-09 21:44:50 +02:00
if f . opt . SSEKMSKeyID != "" {
req . SSEKMSKeyId = & f . opt . SSEKMSKeyID
}
2024-08-03 12:35:32 +02:00
if req . StorageClass == types . StorageClass ( "" ) && f . opt . StorageClass != "" {
req . StorageClass = types . StorageClass ( f . opt . StorageClass )
2019-09-09 21:44:50 +02:00
}
2019-10-04 17:49:06 +02:00
2020-07-30 11:52:32 +02:00
if src . bytes >= int64 ( f . opt . CopyCutoff ) {
return f . copyMultipart ( ctx , req , dstBucket , dstPath , srcBucket , srcPath , src )
2019-10-04 17:49:06 +02:00
}
2019-09-09 21:44:50 +02:00
return f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
_ , err := f . c . CopyObject ( ctx , req )
2021-03-16 16:50:02 +01:00
return f . shouldRetry ( ctx , err )
2019-09-09 21:44:50 +02:00
} )
}
2019-10-04 17:49:06 +02:00
func calculateRange ( partSize , partIndex , numParts , totalSize int64 ) string {
start := partIndex * partSize
var ends string
if partIndex == numParts - 1 {
2019-12-02 18:00:54 +01:00
if totalSize >= 1 {
ends = strconv . FormatInt ( totalSize - 1 , 10 )
2019-10-04 17:49:06 +02:00
}
} else {
ends = strconv . FormatInt ( start + partSize - 1 , 10 )
}
return fmt . Sprintf ( "bytes=%v-%v" , start , ends )
}
2020-07-30 11:52:32 +02:00
func ( f * Fs ) copyMultipart ( ctx context . Context , copyReq * s3 . CopyObjectInput , dstBucket , dstPath , srcBucket , srcPath string , src * Object ) ( err error ) {
info , err := src . headObject ( ctx )
if err != nil {
return err
}
req := & s3 . CreateMultipartUploadInput { }
// Fill in the request from the head info
2022-07-28 12:49:19 +02:00
//structs.SetFrom(req, info)
setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput ( req , info )
2020-07-30 11:52:32 +02:00
// If copy metadata was set then set the Metadata to that read
// from the head request
2024-08-03 12:35:32 +02:00
if copyReq . MetadataDirective == types . MetadataDirectiveCopy {
2020-07-30 11:52:32 +02:00
copyReq . Metadata = info . Metadata
}
// Overwrite any from the copyReq
2022-07-28 12:49:19 +02:00
//structs.SetFrom(req, copyReq)
setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput ( req , copyReq )
2020-07-30 11:52:32 +02:00
req . Bucket = & dstBucket
req . Key = & dstPath
2019-10-04 17:49:06 +02:00
var cout * s3 . CreateMultipartUploadOutput
if err := f . pacer . Call ( func ( ) ( bool , error ) {
var err error
2024-08-03 12:35:32 +02:00
cout , err = f . c . CreateMultipartUpload ( ctx , req )
2021-03-16 16:50:02 +01:00
return f . shouldRetry ( ctx , err )
2019-10-04 17:49:06 +02:00
} ) ; err != nil {
return err
}
uid := cout . UploadId
2020-06-04 12:09:27 +02:00
defer atexit . OnError ( & err , func ( ) {
// Try to abort the upload, but ignore the error.
2020-07-30 11:52:32 +02:00
fs . Debugf ( src , "Cancelling multipart copy" )
2020-06-04 12:09:27 +02:00
_ = f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
_ , err := f . c . AbortMultipartUpload ( context . Background ( ) , & s3 . AbortMultipartUploadInput {
2020-06-04 12:09:27 +02:00
Bucket : & dstBucket ,
Key : & dstPath ,
UploadId : uid ,
RequestPayer : req . RequestPayer ,
2019-10-04 17:49:06 +02:00
} )
2021-03-16 16:50:02 +01:00
return f . shouldRetry ( ctx , err )
2020-06-04 12:09:27 +02:00
} )
} ) ( )
2019-10-04 17:49:06 +02:00
2020-07-30 11:52:32 +02:00
srcSize := src . bytes
2019-12-02 18:14:57 +01:00
partSize := int64 ( f . opt . CopyCutoff )
2019-10-04 17:49:06 +02:00
numParts := ( srcSize - 1 ) / partSize + 1
2020-07-30 11:52:32 +02:00
fs . Debugf ( src , "Starting multipart copy with %d parts" , numParts )
2023-12-01 11:30:44 +01:00
var (
2024-08-03 12:35:32 +02:00
parts = make ( [ ] types . CompletedPart , numParts )
2023-12-01 11:30:44 +01:00
g , gCtx = errgroup . WithContext ( ctx )
)
g . SetLimit ( f . opt . UploadConcurrency )
2024-08-03 12:35:32 +02:00
for partNum := int32 ( 1 ) ; int64 ( partNum ) <= numParts ; partNum ++ {
2023-12-01 11:30:44 +01:00
// Fail fast, in case an errgroup managed function returns an error
// gCtx is cancelled. There is no point in uploading all the other parts.
if gCtx . Err ( ) != nil {
break
}
partNum := partNum // for closure
g . Go ( func ( ) error {
var uout * s3 . UploadPartCopyOutput
2020-07-30 11:52:32 +02:00
uploadPartReq := & s3 . UploadPartCopyInput { }
2022-07-28 12:49:19 +02:00
//structs.SetFrom(uploadPartReq, copyReq)
setFrom_s3UploadPartCopyInput_s3CopyObjectInput ( uploadPartReq , copyReq )
2020-07-30 11:52:32 +02:00
uploadPartReq . Bucket = & dstBucket
uploadPartReq . Key = & dstPath
uploadPartReq . PartNumber = & partNum
uploadPartReq . UploadId = uid
2024-08-03 12:35:32 +02:00
uploadPartReq . CopySourceRange = aws . String ( calculateRange ( partSize , int64 ( partNum - 1 ) , numParts , srcSize ) )
2023-12-01 11:30:44 +01:00
err := f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
uout , err = f . c . UploadPartCopy ( gCtx , uploadPartReq )
2023-12-01 11:30:44 +01:00
return f . shouldRetry ( gCtx , err )
} )
2019-10-04 17:49:06 +02:00
if err != nil {
2023-12-01 11:30:44 +01:00
return err
2019-10-04 17:49:06 +02:00
}
2024-08-03 12:35:32 +02:00
parts [ partNum - 1 ] = types . CompletedPart {
2019-10-04 17:49:06 +02:00
PartNumber : & partNum ,
ETag : uout . CopyPartResult . ETag ,
2023-12-01 11:30:44 +01:00
}
return nil
} )
}
err = g . Wait ( )
if err != nil {
return err
2019-10-04 17:49:06 +02:00
}
return f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
_ , err := f . c . CompleteMultipartUpload ( ctx , & s3 . CompleteMultipartUploadInput {
2019-10-04 17:49:06 +02:00
Bucket : & dstBucket ,
Key : & dstPath ,
2024-08-03 12:35:32 +02:00
MultipartUpload : & types . CompletedMultipartUpload {
2019-10-04 17:49:06 +02:00
Parts : parts ,
} ,
RequestPayer : req . RequestPayer ,
UploadId : uid ,
} )
2021-03-16 16:50:02 +01:00
return f . shouldRetry ( ctx , err )
2019-10-04 17:49:06 +02:00
} )
}
2020-10-13 23:43:40 +02:00
// Copy src to this remote using server-side copy operations.
2015-02-14 19:48:08 +01:00
//
2022-08-05 17:35:41 +02:00
// This is stored with the remote path given.
2015-02-14 19:48:08 +01:00
//
2022-08-05 17:35:41 +02:00
// It returns the destination Object and a possible error.
2015-02-14 19:48:08 +01:00
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Copy ( ctx context . Context , src fs . Object , remote string ) ( fs . Object , error ) {
2022-07-26 18:58:57 +02:00
if f . opt . VersionAt . IsSet ( ) {
return nil , errNotWithVersionAt
}
2019-08-09 12:29:36 +02:00
dstBucket , dstPath := f . split ( remote )
2023-04-26 11:59:17 +02:00
err := f . mkdirParent ( ctx , remote )
2017-06-28 22:14:53 +02:00
if err != nil {
return nil , err
}
2015-11-07 12:14:46 +01:00
srcObj , ok := src . ( * Object )
2015-02-14 19:48:08 +01:00
if ! ok {
2017-02-09 12:01:20 +01:00
fs . Debugf ( src , "Can't copy - not same remote type" )
2015-02-14 19:48:08 +01:00
return nil , fs . ErrorCantCopy
}
2024-03-05 18:21:06 +01:00
2019-08-09 12:29:36 +02:00
srcBucket , srcPath := srcObj . split ( )
2015-02-14 19:48:08 +01:00
req := s3 . CopyObjectInput {
2024-08-03 12:35:32 +02:00
MetadataDirective : types . MetadataDirectiveCopy ,
2015-02-14 19:48:08 +01:00
}
2024-03-05 18:21:06 +01:00
// Update the metadata if it is in use
if ci := fs . GetConfig ( ctx ) ; ci . Metadata {
ui , err := srcObj . prepareUpload ( ctx , src , fs . MetadataAsOpenOptions ( ctx ) , true )
if err != nil {
return nil , fmt . Errorf ( "failed to prepare upload: %w" , err )
}
setFrom_s3CopyObjectInput_s3PutObjectInput ( & req , ui . req )
2024-08-03 12:35:32 +02:00
req . MetadataDirective = types . MetadataDirectiveReplace
2024-03-05 18:21:06 +01:00
}
2020-07-30 11:52:32 +02:00
err = f . copy ( ctx , & req , dstBucket , dstPath , srcBucket , srcPath , srcObj )
2015-02-14 19:48:08 +01:00
if err != nil {
return nil , err
}
2019-06-17 10:34:30 +02:00
return f . NewObject ( ctx , remote )
2015-02-14 19:48:08 +01:00
}
2016-01-11 13:39:33 +01:00
// Hashes returns the supported hash sets.
2018-01-12 17:30:54 +01:00
func ( f * Fs ) Hashes ( ) hash . Set {
2018-01-18 21:27:52 +01:00
return hash . Set ( hash . MD5 )
2016-01-11 13:39:33 +01:00
}
2020-06-04 23:03:12 +02:00
// PublicLink generates a public link to the remote path (usually readable by anyone)
func ( f * Fs ) PublicLink ( ctx context . Context , remote string , expire fs . Duration , unlink bool ) ( link string , err error ) {
2020-06-18 18:50:50 +02:00
if strings . HasSuffix ( remote , "/" ) {
return "" , fs . ErrorCantShareDirectories
}
2022-07-25 17:06:15 +02:00
obj , err := f . NewObject ( ctx , remote )
if err != nil {
2020-06-04 23:03:12 +02:00
return "" , err
}
2022-07-25 17:06:15 +02:00
o := obj . ( * Object )
2020-06-18 18:50:50 +02:00
if expire > maxExpireDuration {
fs . Logf ( f , "Public Link: Reducing expiry to %v as %v is greater than the max time allowed" , maxExpireDuration , expire )
expire = maxExpireDuration
}
2024-08-03 12:35:32 +02:00
bucket , bucketPath := f . split ( remote )
httpReq , err := s3 . NewPresignClient ( f . c ) . PresignGetObject ( ctx , & s3 . GetObjectInput {
2022-07-25 17:06:15 +02:00
Bucket : & bucket ,
Key : & bucketPath ,
VersionId : o . versionID ,
2024-08-03 12:35:32 +02:00
} , s3 . WithPresignExpires ( time . Duration ( expire ) ) )
if err != nil {
return "" , err
}
return httpReq . URL , nil
2020-06-04 23:03:12 +02:00
}
2020-06-24 12:02:34 +02:00
var commandHelp = [ ] fs . CommandHelp { {
Name : "restore" ,
Short : "Restore objects from GLACIER to normal storage" ,
Long : ` This command can be used to restore one or more objects from GLACIER
to normal storage .
Usage Examples :
2023-07-10 19:12:54 +02:00
rclone backend restore s3 : bucket / path / to / object - o priority = PRIORITY - o lifetime = DAYS
rclone backend restore s3 : bucket / path / to / directory - o priority = PRIORITY - o lifetime = DAYS
rclone backend restore s3 : bucket - o priority = PRIORITY - o lifetime = DAYS
2020-06-24 12:02:34 +02:00
2023-01-20 21:47:36 +01:00
This flag also obeys the filters . Test first with -- interactive / - i or -- dry - run flags
2020-06-24 12:02:34 +02:00
2023-07-10 19:12:54 +02:00
rclone -- interactive backend restore -- include "*.txt" s3 : bucket / path - o priority = Standard - o lifetime = 1
2020-06-24 12:02:34 +02:00
All the objects shown will be marked for restore , then
2023-07-10 19:12:54 +02:00
rclone backend restore -- include "*.txt" s3 : bucket / path - o priority = Standard - o lifetime = 1
2020-06-24 12:02:34 +02:00
It returns a list of status dictionaries with Remote and Status
Spelling fixes
Fix spelling of: above, already, anonymous, associated,
authentication, bandwidth, because, between, blocks, calculate,
candidates, cautious, changelog, cleaner, clipboard, command,
completely, concurrently, considered, constructs, corrupt, current,
daemon, dependencies, deprecated, directory, dispatcher, download,
eligible, ellipsis, encrypter, endpoint, entrieslist, essentially,
existing writers, existing, expires, filesystem, flushing, frequently,
hierarchy, however, implementation, implements, inaccurate,
individually, insensitive, longer, maximum, metadata, modified,
multipart, namedirfirst, nextcloud, obscured, opened, optional,
owncloud, pacific, passphrase, password, permanently, persimmon,
positive, potato, protocol, quota, receiving, recommends, referring,
requires, revisited, satisfied, satisfies, satisfy, semver,
serialized, session, storage, strategies, stringlist, successful,
supported, surprise, temporarily, temporary, transactions, unneeded,
update, uploads, wrapped
Signed-off-by: Josh Soref <jsoref@users.noreply.github.com>
2020-10-09 02:17:24 +02:00
keys . The Status will be OK if it was successful or an error message
2020-06-24 12:02:34 +02:00
if not .
[
{
"Status" : "OK" ,
2023-09-06 13:42:03 +02:00
"Remote" : "test.txt"
2020-06-24 12:02:34 +02:00
} ,
{
"Status" : "OK" ,
2023-09-06 13:42:03 +02:00
"Remote" : "test/file4.txt"
2020-06-24 12:02:34 +02:00
}
]
` ,
Opts : map [ string ] string {
"priority" : "Priority of restore: Standard|Expedited|Bulk" ,
"lifetime" : "Lifetime of the active copy in days" ,
"description" : "The optional description for the job." ,
} ,
2023-09-06 13:42:03 +02:00
} , {
Name : "restore-status" ,
Short : "Show the restore status for objects being restored from GLACIER to normal storage" ,
Long : ` This command can be used to show the status for objects being restored from GLACIER
to normal storage .
Usage Examples :
rclone backend restore - status s3 : bucket / path / to / object
rclone backend restore - status s3 : bucket / path / to / directory
rclone backend restore - status - o all s3 : bucket / path / to / directory
This command does not obey the filters .
It returns a list of status dictionaries .
[
{
"Remote" : "file.txt" ,
"VersionID" : null ,
"RestoreStatus" : {
"IsRestoreInProgress" : true ,
"RestoreExpiryDate" : "2023-09-06T12:29:19+01:00"
} ,
"StorageClass" : "GLACIER"
} ,
{
"Remote" : "test.pdf" ,
"VersionID" : null ,
"RestoreStatus" : {
"IsRestoreInProgress" : false ,
"RestoreExpiryDate" : "2023-09-06T12:29:19+01:00"
} ,
"StorageClass" : "DEEP_ARCHIVE"
}
]
` ,
Opts : map [ string ] string {
"all" : "if set then show all objects, not just ones with restore status" ,
} ,
2020-06-25 17:11:05 +02:00
} , {
Name : "list-multipart-uploads" ,
Short : "List the unfinished multipart uploads" ,
Long : ` This command lists the unfinished multipart uploads in JSON format .
rclone backend list - multipart s3 : bucket / path / to / object
It returns a dictionary of buckets with values as lists of unfinished
multipart uploads .
You can call it with no bucket in which case it lists all bucket , with
a bucket or with a bucket and path .
{
"rclone" : [
{
"Initiated" : "2020-06-26T14:20:36Z" ,
"Initiator" : {
"DisplayName" : "XXX" ,
"ID" : "arn:aws:iam::XXX:user/XXX"
} ,
"Key" : "KEY" ,
"Owner" : {
"DisplayName" : null ,
"ID" : "XXX"
} ,
"StorageClass" : "STANDARD" ,
"UploadId" : "XXX"
}
] ,
"rclone-1000files" : [ ] ,
"rclone-dst" : [ ]
}
` ,
} , {
Name : "cleanup" ,
Short : "Remove unfinished multipart uploads." ,
Long : ` This command removes unfinished multipart uploads of age greater than
max - age which defaults to 24 hours .
2023-01-20 21:47:36 +01:00
Note that you can use -- interactive / - i or -- dry - run with this command to see what
it would do .
2020-06-25 17:11:05 +02:00
rclone backend cleanup s3 : bucket / path / to / object
rclone backend cleanup - o max - age = 7 w s3 : bucket / path / to / object
Durations are parsed as per the rest of rclone , 2 h , 7 d , 7 w etc .
` ,
Opts : map [ string ] string {
"max-age" : "Max age of upload to delete" ,
} ,
2022-07-26 16:03:32 +02:00
} , {
Name : "cleanup-hidden" ,
Short : "Remove old versions of files." ,
Long : ` This command removes any old hidden versions of files
on a versions enabled bucket .
2023-01-20 21:47:36 +01:00
Note that you can use -- interactive / - i or -- dry - run with this command to see what
it would do .
2022-07-26 16:03:32 +02:00
rclone backend cleanup - hidden s3 : bucket / path / to / dir
` ,
2022-07-25 17:05:17 +02:00
} , {
Name : "versioning" ,
Short : "Set/get versioning support for a bucket." ,
Long : ` This command sets versioning support if a parameter is
passed and then returns the current versioning status for the bucket
supplied .
rclone backend versioning s3 : bucket # read status only
rclone backend versioning s3 : bucket Enabled
rclone backend versioning s3 : bucket Suspended
It may return "Enabled" , "Suspended" or "Unversioned" . Note that once versioning
has been enabled the status can ' t be set back to "Unversioned" .
` ,
2023-07-15 19:56:58 +02:00
} , {
Name : "set" ,
Short : "Set command for updating the config parameters." ,
Long : ` This set command can be used to update the config parameters
for a running s3 backend .
Usage Examples :
rclone backend set s3 : [ - o opt_name = opt_value ] [ - o opt_name2 = opt_value2 ]
rclone rc backend / command command = set fs = s3 : [ - o opt_name = opt_value ] [ - o opt_name2 = opt_value2 ]
rclone rc backend / command command = set fs = s3 : - o session_token = X - o access_key_id = X - o secret_access_key = X
The option keys are named as they are in the config file .
This rebuilds the connection to the s3 backend when it is called with
the new parameters . Only new parameters need be passed as the values
will default to those currently in use .
It doesn ' t return anything .
` ,
2020-06-24 12:02:34 +02:00
} }
// Command the backend to run a named command
//
// The command run is name
// args may be used to read arguments from
// opts may be used to read optional arguments from
//
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
func ( f * Fs ) Command ( ctx context . Context , name string , arg [ ] string , opt map [ string ] string ) ( out interface { } , err error ) {
switch name {
case "restore" :
req := s3 . RestoreObjectInput {
//Bucket: &f.rootBucket,
//Key: &encodedDirectory,
2024-08-03 12:35:32 +02:00
RestoreRequest : & types . RestoreRequest { } ,
2020-06-24 12:02:34 +02:00
}
if lifetime := opt [ "lifetime" ] ; lifetime != "" {
ilifetime , err := strconv . ParseInt ( lifetime , 10 , 64 )
2024-08-03 12:35:32 +02:00
ilifetime32 := int32 ( ilifetime )
2020-06-24 12:02:34 +02:00
if err != nil {
2021-11-04 11:12:57 +01:00
return nil , fmt . Errorf ( "bad lifetime: %w" , err )
2020-06-24 12:02:34 +02:00
}
2024-08-03 12:35:32 +02:00
req . RestoreRequest . Days = & ilifetime32
2020-06-24 12:02:34 +02:00
}
if priority := opt [ "priority" ] ; priority != "" {
2024-08-03 12:35:32 +02:00
req . RestoreRequest . GlacierJobParameters = & types . GlacierJobParameters {
Tier : types . Tier ( priority ) ,
2020-06-24 12:02:34 +02:00
}
}
if description := opt [ "description" ] ; description != "" {
req . RestoreRequest . Description = & description
}
type status struct {
Status string
Remote string
}
var (
outMu sync . Mutex
out = [ ] status { }
)
err = operations . ListFn ( ctx , f , func ( obj fs . Object ) {
// Remember this is run --checkers times concurrently
o , ok := obj . ( * Object )
st := status { Status : "OK" , Remote : obj . Remote ( ) }
defer func ( ) {
outMu . Lock ( )
out = append ( out , st )
outMu . Unlock ( )
} ( )
if operations . SkipDestructive ( ctx , obj , "restore" ) {
return
}
if ! ok {
st . Status = "Not an S3 object"
return
}
2022-05-24 13:32:39 +02:00
if o . storageClass == nil || ( * o . storageClass != "GLACIER" && * o . storageClass != "DEEP_ARCHIVE" ) {
2022-05-12 21:42:37 +02:00
st . Status = "Not GLACIER or DEEP_ARCHIVE storage class"
return
}
2020-06-24 12:02:34 +02:00
bucket , bucketPath := o . split ( )
reqCopy := req
reqCopy . Bucket = & bucket
reqCopy . Key = & bucketPath
2022-07-25 17:06:15 +02:00
reqCopy . VersionId = o . versionID
2020-06-24 12:02:34 +02:00
err = f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
_ , err = f . c . RestoreObject ( ctx , & reqCopy )
2021-03-16 16:50:02 +01:00
return f . shouldRetry ( ctx , err )
2020-06-24 12:02:34 +02:00
} )
if err != nil {
st . Status = err . Error ( )
}
} )
if err != nil {
return out , err
}
return out , nil
2023-09-06 13:42:03 +02:00
case "restore-status" :
_ , all := opt [ "all" ]
return f . restoreStatus ( ctx , all )
2020-06-25 17:11:05 +02:00
case "list-multipart-uploads" :
return f . listMultipartUploadsAll ( ctx )
case "cleanup" :
maxAge := 24 * time . Hour
if opt [ "max-age" ] != "" {
maxAge , err = fs . ParseDuration ( opt [ "max-age" ] )
if err != nil {
2021-11-04 11:12:57 +01:00
return nil , fmt . Errorf ( "bad max-age: %w" , err )
2020-06-25 17:11:05 +02:00
}
}
return nil , f . cleanUp ( ctx , maxAge )
2022-07-26 16:03:32 +02:00
case "cleanup-hidden" :
return nil , f . CleanUpHidden ( ctx )
2022-07-25 17:05:17 +02:00
case "versioning" :
return f . setGetVersioning ( ctx , arg ... )
2023-07-15 19:56:58 +02:00
case "set" :
newOpt := f . opt
err := configstruct . Set ( configmap . Simple ( opt ) , & newOpt )
if err != nil {
return nil , fmt . Errorf ( "reading config: %w" , err )
}
2024-08-03 12:35:32 +02:00
c , err := s3Connection ( f . ctx , & newOpt , f . srv )
2023-07-15 19:56:58 +02:00
if err != nil {
return nil , fmt . Errorf ( "updating session: %w" , err )
}
f . c = c
f . opt = newOpt
keys := [ ] string { }
for k := range opt {
keys = append ( keys , k )
}
fs . Logf ( f , "Updated config values: %s" , strings . Join ( keys , ", " ) )
return nil , nil
2020-06-24 12:02:34 +02:00
default :
return nil , fs . ErrorCommandNotFound
}
}
2023-09-06 13:42:03 +02:00
// Returned from "restore-status"
type restoreStatusOut struct {
Remote string
VersionID * string
2024-08-03 12:35:32 +02:00
RestoreStatus * types . RestoreStatus
StorageClass types . ObjectStorageClass
2023-09-06 13:42:03 +02:00
}
// Recursively enumerate the current fs to find objects with a restore status
func ( f * Fs ) restoreStatus ( ctx context . Context , all bool ) ( out [ ] restoreStatusOut , err error ) {
fs . Debugf ( f , "all = %v" , all )
bucket , directory := f . split ( "" )
out = [ ] restoreStatusOut { }
err = f . list ( ctx , listOpt {
bucket : bucket ,
directory : directory ,
prefix : f . rootDirectory ,
addBucket : f . rootBucket == "" ,
recurse : true ,
withVersions : f . opt . Versions ,
versionAt : f . opt . VersionAt ,
2023-11-08 16:29:23 +01:00
hidden : f . opt . VersionDeleted ,
2023-09-06 13:42:03 +02:00
restoreStatus : true ,
2024-08-03 12:35:32 +02:00
} , func ( remote string , object * types . Object , versionID * string , isDirectory bool ) error {
2023-09-06 13:42:03 +02:00
entry , err := f . itemToDirEntry ( ctx , remote , object , versionID , isDirectory )
if err != nil {
return err
}
if entry != nil {
if o , ok := entry . ( * Object ) ; ok && ( all || object . RestoreStatus != nil ) {
out = append ( out , restoreStatusOut {
Remote : o . remote ,
VersionID : o . versionID ,
RestoreStatus : object . RestoreStatus ,
StorageClass : object . StorageClass ,
} )
}
}
return nil
} )
if err != nil {
return nil , err
}
// bucket must be present if listing succeeded
f . cache . MarkOK ( bucket )
return out , nil
}
2020-06-25 17:11:05 +02:00
// listMultipartUploads lists all outstanding multipart uploads for (bucket, key)
//
// Note that rather lazily we treat key as a prefix so it matches
Spelling fixes
Fix spelling of: above, already, anonymous, associated,
authentication, bandwidth, because, between, blocks, calculate,
candidates, cautious, changelog, cleaner, clipboard, command,
completely, concurrently, considered, constructs, corrupt, current,
daemon, dependencies, deprecated, directory, dispatcher, download,
eligible, ellipsis, encrypter, endpoint, entrieslist, essentially,
existing writers, existing, expires, filesystem, flushing, frequently,
hierarchy, however, implementation, implements, inaccurate,
individually, insensitive, longer, maximum, metadata, modified,
multipart, namedirfirst, nextcloud, obscured, opened, optional,
owncloud, pacific, passphrase, password, permanently, persimmon,
positive, potato, protocol, quota, receiving, recommends, referring,
requires, revisited, satisfied, satisfies, satisfy, semver,
serialized, session, storage, strategies, stringlist, successful,
supported, surprise, temporarily, temporary, transactions, unneeded,
update, uploads, wrapped
Signed-off-by: Josh Soref <jsoref@users.noreply.github.com>
2020-10-09 02:17:24 +02:00
// directories and objects. This could surprise the user if they ask
2020-06-25 17:11:05 +02:00
// for "dir" and it returns "dirKey"
2024-08-03 12:35:32 +02:00
func ( f * Fs ) listMultipartUploads ( ctx context . Context , bucket , key string ) ( uploads [ ] types . MultipartUpload , err error ) {
2020-06-25 17:11:05 +02:00
var (
keyMarker * string
uploadIDMarker * string
)
2024-08-03 12:35:32 +02:00
uploads = [ ] types . MultipartUpload { }
2020-06-25 17:11:05 +02:00
for {
req := s3 . ListMultipartUploadsInput {
Bucket : & bucket ,
MaxUploads : & f . opt . ListChunk ,
KeyMarker : keyMarker ,
UploadIdMarker : uploadIDMarker ,
Prefix : & key ,
}
var resp * s3 . ListMultipartUploadsOutput
err = f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
resp , err = f . c . ListMultipartUploads ( ctx , & req )
2021-03-16 16:50:02 +01:00
return f . shouldRetry ( ctx , err )
2020-06-25 17:11:05 +02:00
} )
if err != nil {
2021-11-04 11:12:57 +01:00
return nil , fmt . Errorf ( "list multipart uploads bucket %q key %q: %w" , bucket , key , err )
2020-06-25 17:11:05 +02:00
}
uploads = append ( uploads , resp . Uploads ... )
2024-08-03 12:35:32 +02:00
if ! deref ( resp . IsTruncated ) {
2020-06-25 17:11:05 +02:00
break
}
keyMarker = resp . NextKeyMarker
uploadIDMarker = resp . NextUploadIdMarker
}
return uploads , nil
}
2024-08-03 12:35:32 +02:00
func ( f * Fs ) listMultipartUploadsAll ( ctx context . Context ) ( uploadsMap map [ string ] [ ] types . MultipartUpload , err error ) {
uploadsMap = make ( map [ string ] [ ] types . MultipartUpload )
2020-06-25 17:11:05 +02:00
bucket , directory := f . split ( "" )
if bucket != "" {
uploads , err := f . listMultipartUploads ( ctx , bucket , directory )
if err != nil {
return uploadsMap , err
}
uploadsMap [ bucket ] = uploads
return uploadsMap , nil
}
entries , err := f . listBuckets ( ctx )
if err != nil {
return uploadsMap , err
}
for _ , entry := range entries {
bucket := entry . Remote ( )
uploads , listErr := f . listMultipartUploads ( ctx , bucket , "" )
if listErr != nil {
err = listErr
fs . Errorf ( f , "%v" , err )
}
uploadsMap [ bucket ] = uploads
}
return uploadsMap , err
}
// cleanUpBucket removes all pending multipart uploads for a given bucket over the age of maxAge
2024-08-03 12:35:32 +02:00
func ( f * Fs ) cleanUpBucket ( ctx context . Context , bucket string , maxAge time . Duration , uploads [ ] types . MultipartUpload ) ( err error ) {
2020-06-25 17:11:05 +02:00
fs . Infof ( f , "cleaning bucket %q of pending multipart uploads older than %v" , bucket , maxAge )
for _ , upload := range uploads {
if upload . Initiated != nil && upload . Key != nil && upload . UploadId != nil {
age := time . Since ( * upload . Initiated )
what := fmt . Sprintf ( "pending multipart upload for bucket %q key %q dated %v (%v ago)" , bucket , * upload . Key , upload . Initiated , age )
if age > maxAge {
fs . Infof ( f , "removing %s" , what )
if operations . SkipDestructive ( ctx , what , "remove pending upload" ) {
continue
}
req := s3 . AbortMultipartUploadInput {
Bucket : & bucket ,
UploadId : upload . UploadId ,
Key : upload . Key ,
}
2024-08-03 12:35:32 +02:00
_ , abortErr := f . c . AbortMultipartUpload ( ctx , & req )
2020-06-25 17:11:05 +02:00
if abortErr != nil {
2021-11-04 11:12:57 +01:00
err = fmt . Errorf ( "failed to remove %s: %w" , what , abortErr )
2020-06-25 17:11:05 +02:00
fs . Errorf ( f , "%v" , err )
}
} else {
fs . Debugf ( f , "ignoring %s" , what )
}
}
}
return err
}
// CleanUp removes all pending multipart uploads
func ( f * Fs ) cleanUp ( ctx context . Context , maxAge time . Duration ) ( err error ) {
uploadsMap , err := f . listMultipartUploadsAll ( ctx )
if err != nil {
return err
}
for bucket , uploads := range uploadsMap {
cleanErr := f . cleanUpBucket ( ctx , bucket , maxAge , uploads )
if err != nil {
fs . Errorf ( f , "Failed to cleanup bucket %q: %v" , bucket , cleanErr )
err = cleanErr
}
}
return err
}
2022-07-25 17:05:17 +02:00
// Read whether the bucket is versioned or not
func ( f * Fs ) isVersioned ( ctx context . Context ) bool {
f . versioningMu . Lock ( )
defer f . versioningMu . Unlock ( )
if ! f . versioning . Valid {
_ , _ = f . setGetVersioning ( ctx )
fs . Debugf ( f , "bucket is versioned: %v" , f . versioning . Value )
}
return f . versioning . Value
}
// Set or get bucket versioning.
//
// Pass no arguments to get, or pass "Enabled" or "Suspended"
//
// Updates f.versioning
2024-08-03 12:35:32 +02:00
func ( f * Fs ) setGetVersioning ( ctx context . Context , arg ... string ) ( status types . BucketVersioningStatus , err error ) {
2022-07-25 17:05:17 +02:00
if len ( arg ) > 1 {
return "" , errors . New ( "too many arguments" )
}
if f . rootBucket == "" {
return "" , errors . New ( "need a bucket" )
}
if len ( arg ) == 1 {
2024-08-03 12:35:32 +02:00
var versioning = types . VersioningConfiguration {
Status : types . BucketVersioningStatus ( arg [ 0 ] ) ,
2022-07-25 17:05:17 +02:00
}
// Disabled is indicated by the parameter missing
2024-08-03 12:35:32 +02:00
if versioning . Status == types . BucketVersioningStatus ( "Disabled" ) {
versioning . Status = types . BucketVersioningStatus ( "" )
2022-07-25 17:05:17 +02:00
}
req := s3 . PutBucketVersioningInput {
Bucket : & f . rootBucket ,
VersioningConfiguration : & versioning ,
}
err := f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
_ , err = f . c . PutBucketVersioning ( ctx , & req )
2022-07-25 17:05:17 +02:00
return f . shouldRetry ( ctx , err )
} )
if err != nil {
return "" , err
}
}
req := s3 . GetBucketVersioningInput {
Bucket : & f . rootBucket ,
}
var resp * s3 . GetBucketVersioningOutput
err = f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
resp , err = f . c . GetBucketVersioning ( ctx , & req )
2022-07-25 17:05:17 +02:00
return f . shouldRetry ( ctx , err )
} )
f . versioning . Valid = true
f . versioning . Value = false
if err != nil {
fs . Errorf ( f , "Failed to read versioning status, assuming unversioned: %v" , err )
return "" , err
}
2024-08-03 12:35:32 +02:00
if len ( resp . Status ) == 0 {
2022-07-25 17:05:17 +02:00
return "Unversioned" , err
}
f . versioning . Value = true
2024-08-03 12:35:32 +02:00
return resp . Status , err
2022-07-25 17:05:17 +02:00
}
2020-06-25 17:11:05 +02:00
// CleanUp removes all pending multipart uploads older than 24 hours
func ( f * Fs ) CleanUp ( ctx context . Context ) ( err error ) {
return f . cleanUp ( ctx , 24 * time . Hour )
}
2022-07-26 16:03:32 +02:00
// purge deletes all the files and directories
//
// if oldOnly is true then it deletes only non current files.
//
// Implemented here so we can make sure we delete old versions.
func ( f * Fs ) purge ( ctx context . Context , dir string , oldOnly bool ) error {
2022-07-26 18:58:57 +02:00
if f . opt . VersionAt . IsSet ( ) {
return errNotWithVersionAt
}
2022-07-26 16:03:32 +02:00
bucket , directory := f . split ( dir )
if bucket == "" {
return errors . New ( "can't purge from root" )
}
versioned := f . isVersioned ( ctx )
if ! versioned && oldOnly {
fs . Infof ( f , "bucket is not versioned so not removing old versions" )
return nil
}
var errReturn error
var checkErrMutex sync . Mutex
var checkErr = func ( err error ) {
if err == nil {
return
}
checkErrMutex . Lock ( )
defer checkErrMutex . Unlock ( )
if errReturn == nil {
errReturn = err
}
}
// Delete Config.Transfers in parallel
delChan := make ( fs . ObjectsChan , f . ci . Transfers )
delErr := make ( chan error , 1 )
go func ( ) {
delErr <- operations . DeleteFiles ( ctx , delChan )
} ( )
2022-07-27 11:46:28 +02:00
checkErr ( f . list ( ctx , listOpt {
2023-02-27 16:32:59 +01:00
bucket : bucket ,
directory : directory ,
prefix : f . rootDirectory ,
addBucket : f . rootBucket == "" ,
recurse : true ,
withVersions : versioned ,
hidden : true ,
noSkipMarkers : true ,
2024-08-03 12:35:32 +02:00
} , func ( remote string , object * types . Object , versionID * string , isDirectory bool ) error {
2022-07-26 16:03:32 +02:00
if isDirectory {
return nil
}
2023-08-25 18:39:16 +02:00
// If the root is a dirmarker it will have lost its trailing /
if remote == "" {
remote = "/"
}
2022-07-26 16:03:32 +02:00
oi , err := f . newObjectWithInfo ( ctx , remote , object , versionID )
if err != nil {
fs . Errorf ( object , "Can't create object %+v" , err )
return nil
}
2023-02-06 11:30:22 +01:00
tr := accounting . Stats ( ctx ) . NewCheckingTransfer ( oi , "checking" )
2022-07-26 16:03:32 +02:00
// Work out whether the file is the current version or not
isCurrentVersion := ! versioned || ! version . Match ( remote )
fs . Debugf ( nil , "%q version %v" , remote , version . Match ( remote ) )
if oldOnly && isCurrentVersion {
// Check current version of the file
if object . Size == isDeleteMarker {
2024-08-03 12:35:32 +02:00
fs . Debugf ( remote , "Deleting current version (id %q) as it is a delete marker" , deref ( versionID ) )
2022-07-26 16:03:32 +02:00
delChan <- oi
} else {
2024-08-03 12:35:32 +02:00
fs . Debugf ( remote , "Not deleting current version %q" , deref ( versionID ) )
2022-07-26 16:03:32 +02:00
}
} else {
if object . Size == isDeleteMarker {
2024-08-03 12:35:32 +02:00
fs . Debugf ( remote , "Deleting delete marker (id %q)" , deref ( versionID ) )
2022-07-26 16:03:32 +02:00
} else {
2024-08-03 12:35:32 +02:00
fs . Debugf ( remote , "Deleting (id %q)" , deref ( versionID ) )
2022-07-26 16:03:32 +02:00
}
delChan <- oi
}
tr . Done ( ctx , nil )
return nil
} ) )
close ( delChan )
checkErr ( <- delErr )
if ! oldOnly {
checkErr ( f . Rmdir ( ctx , dir ) )
}
return errReturn
}
// Purge deletes all the files and directories including the old versions.
func ( f * Fs ) Purge ( ctx context . Context , dir string ) error {
return f . purge ( ctx , dir , false )
}
// CleanUpHidden deletes all the hidden files.
func ( f * Fs ) CleanUpHidden ( ctx context . Context ) error {
return f . purge ( ctx , "" , true )
}
2013-01-08 19:53:35 +01:00
// ------------------------------------------------------------
2015-09-22 19:47:16 +02:00
// Fs returns the parent Fs
2016-02-18 12:35:25 +01:00
func ( o * Object ) Fs ( ) fs . Info {
2015-11-07 12:14:46 +01:00
return o . fs
2014-03-28 18:56:04 +01:00
}
// Return a string version
2015-11-07 12:14:46 +01:00
func ( o * Object ) String ( ) string {
2014-03-28 18:56:04 +01:00
if o == nil {
return "<nil>"
}
return o . remote
}
2015-09-22 19:47:16 +02:00
// Remote returns the remote path
2015-11-07 12:14:46 +01:00
func ( o * Object ) Remote ( ) string {
2013-06-27 21:13:07 +02:00
return o . remote
2013-01-08 19:53:35 +01:00
}
2015-05-09 11:37:43 +02:00
var matchMd5 = regexp . MustCompile ( ` ^[0-9a-f] { 32}$ ` )
2020-11-23 12:23:06 +01:00
// Set the MD5 from the etag
func ( o * Object ) setMD5FromEtag ( etag string ) {
if o . fs . etagIsNotMD5 {
o . md5 = ""
return
}
if etag == "" {
o . md5 = ""
return
}
hash := strings . Trim ( strings . ToLower ( etag ) , ` " ` )
// Check the etag is a valid md5sum
if ! matchMd5 . MatchString ( hash ) {
o . md5 = ""
return
}
o . md5 = hash
}
2016-01-11 13:39:33 +01:00
// Hash returns the Md5sum of an object returning a lowercase hex string
2019-06-17 10:34:30 +02:00
func ( o * Object ) Hash ( ctx context . Context , t hash . Type ) ( string , error ) {
2018-01-18 21:27:52 +01:00
if t != hash . MD5 {
return "" , hash . ErrUnsupported
2016-01-11 13:39:33 +01:00
}
2022-07-29 18:01:59 +02:00
// If decompressing, erase the hash
if o . bytes < 0 {
return "" , nil
}
2020-11-23 12:23:06 +01:00
// If we haven't got an MD5, then check the metadata
if o . md5 == "" {
2019-06-17 10:34:30 +02:00
err := o . readMetaData ( ctx )
2018-01-06 15:30:10 +01:00
if err != nil {
return "" , err
}
2015-05-09 11:37:43 +02:00
}
2020-11-23 12:23:06 +01:00
return o . md5 , nil
2013-01-08 19:53:35 +01:00
}
// Size returns the size of an object in bytes
2015-11-07 12:14:46 +01:00
func ( o * Object ) Size ( ) int64 {
2013-06-27 21:13:07 +02:00
return o . bytes
2013-01-08 19:53:35 +01:00
}
2020-07-30 11:52:32 +02:00
func ( o * Object ) headObject ( ctx context . Context ) ( resp * s3 . HeadObjectOutput , err error ) {
2019-08-09 12:29:36 +02:00
bucket , bucketPath := o . split ( )
2014-12-23 13:09:02 +01:00
req := s3 . HeadObjectInput {
2022-07-25 17:06:15 +02:00
Bucket : & bucket ,
Key : & bucketPath ,
VersionId : o . versionID ,
2014-07-28 23:32:15 +02:00
}
2023-04-26 11:59:17 +02:00
return o . fs . headObject ( ctx , & req )
}
func ( f * Fs ) headObject ( ctx context . Context , req * s3 . HeadObjectInput ) ( resp * s3 . HeadObjectOutput , err error ) {
if f . opt . RequesterPays {
2024-08-03 12:35:32 +02:00
req . RequestPayer = types . RequestPayerRequester
2020-12-03 03:30:06 +01:00
}
2023-04-26 11:59:17 +02:00
if f . opt . SSECustomerAlgorithm != "" {
req . SSECustomerAlgorithm = & f . opt . SSECustomerAlgorithm
2020-11-20 12:15:48 +01:00
}
2024-08-07 11:14:58 +02:00
if f . opt . SSECustomerKeyBase64 != "" {
req . SSECustomerKey = & f . opt . SSECustomerKeyBase64
2020-11-20 12:15:48 +01:00
}
2023-04-26 11:59:17 +02:00
if f . opt . SSECustomerKeyMD5 != "" {
req . SSECustomerKeyMD5 = & f . opt . SSECustomerKeyMD5
2020-11-20 12:15:48 +01:00
}
2023-04-26 11:59:17 +02:00
err = f . pacer . Call ( func ( ) ( bool , error ) {
2018-09-03 06:41:04 +02:00
var err error
2024-08-03 12:35:32 +02:00
resp , err = f . c . HeadObject ( ctx , req )
2023-04-26 11:59:17 +02:00
return f . shouldRetry ( ctx , err )
2018-09-03 06:41:04 +02:00
} )
2013-01-08 19:53:35 +01:00
if err != nil {
2024-08-03 12:35:32 +02:00
if statusCode := getHTTPStatusCode ( err ) ; statusCode == http . StatusNotFound || statusCode == http . StatusMethodNotAllowed {
return nil , fs . ErrorObjectNotFound
2016-06-25 22:23:20 +02:00
}
2020-07-30 11:52:32 +02:00
return nil , err
2013-01-08 19:53:35 +01:00
}
2023-04-26 11:59:17 +02:00
if req . Bucket != nil {
f . cache . MarkOK ( * req . Bucket )
}
2020-07-30 11:52:32 +02:00
return resp , nil
}
// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
func ( o * Object ) readMetaData ( ctx context . Context ) ( err error ) {
if o . meta != nil {
return nil
}
resp , err := o . headObject ( ctx )
if err != nil {
return err
}
2022-05-24 13:32:39 +02:00
o . setMetaData ( resp )
// resp.ETag, resp.ContentLength, resp.LastModified, resp.Metadata, resp.ContentType, resp.StorageClass)
2021-04-28 12:05:54 +02:00
return nil
}
2024-08-03 12:35:32 +02:00
// Convert S3 metadata into a map[string]string while lowercasing the
// keys
func s3MetadataToMap ( s3Meta map [ string ] string ) map [ string ] string {
2022-06-22 16:40:30 +02:00
meta := make ( map [ string ] string , len ( s3Meta ) )
for k , v := range s3Meta {
2024-08-03 12:35:32 +02:00
meta [ strings . ToLower ( k ) ] = v
2022-06-22 16:40:30 +02:00
}
return meta
}
// Convert our metadata back into S3 metadata
2024-08-03 12:35:32 +02:00
func mapToS3Metadata ( meta map [ string ] string ) map [ string ] string {
return meta
2022-06-22 16:40:30 +02:00
}
2022-05-24 13:32:39 +02:00
func ( o * Object ) setMetaData ( resp * s3 . HeadObjectOutput ) {
2014-05-16 17:27:53 +02:00
// Ignore missing Content-Length assuming it is 0
// Some versions of ceph do this due their apache proxies
2022-05-24 13:32:39 +02:00
if resp . ContentLength != nil {
o . bytes = * resp . ContentLength
2013-01-08 19:53:35 +01:00
}
2024-08-03 12:35:32 +02:00
o . setMD5FromEtag ( deref ( resp . ETag ) )
2022-05-24 13:32:39 +02:00
o . meta = s3MetadataToMap ( resp . Metadata )
2020-11-23 12:23:06 +01:00
// Read MD5 from metadata if present
if md5sumBase64 , ok := o . meta [ metaMD5Hash ] ; ok {
2022-06-22 16:40:30 +02:00
md5sumBytes , err := base64 . StdEncoding . DecodeString ( md5sumBase64 )
2020-11-23 12:23:06 +01:00
if err != nil {
2022-06-22 16:40:30 +02:00
fs . Debugf ( o , "Failed to read md5sum from metadata %q: %v" , md5sumBase64 , err )
2020-11-23 12:23:06 +01:00
} else if len ( md5sumBytes ) != 16 {
2022-06-22 16:40:30 +02:00
fs . Debugf ( o , "Failed to read md5sum from metadata %q: wrong length" , md5sumBase64 )
2020-11-23 12:23:06 +01:00
} else {
o . md5 = hex . EncodeToString ( md5sumBytes )
}
}
2022-05-24 13:32:39 +02:00
if resp . LastModified == nil {
2013-06-27 21:13:07 +02:00
o . lastModified = time . Now ( )
2022-02-16 18:50:11 +01:00
fs . Logf ( o , "Failed to read last modified" )
2014-12-23 13:09:02 +01:00
} else {
2022-10-08 12:27:17 +02:00
// Try to keep the maximum precision in lastModified. If we read
// it from listings then it may have millisecond precision, but
// if we read it from a HEAD/GET request then it will have
// second precision.
2024-05-31 15:14:40 +02:00
equalToWithinOneSecond := o . lastModified . Truncate ( time . Second ) . Equal ( resp . LastModified . Truncate ( time . Second ) )
newHasNs := resp . LastModified . Nanosecond ( ) != 0
2022-10-08 12:27:17 +02:00
if ! equalToWithinOneSecond || newHasNs {
o . lastModified = * resp . LastModified
}
2013-01-08 19:53:35 +01:00
}
2024-08-03 12:35:32 +02:00
o . mimeType = deref ( resp . ContentType )
2022-05-24 13:32:39 +02:00
// Set system metadata
2024-08-03 12:35:32 +02:00
o . storageClass = ( * string ) ( & resp . StorageClass )
2022-05-24 13:32:39 +02:00
o . cacheControl = resp . CacheControl
o . contentDisposition = resp . ContentDisposition
o . contentEncoding = resp . ContentEncoding
o . contentLanguage = resp . ContentLanguage
2022-07-29 18:01:59 +02:00
// If decompressing then size and md5sum are unknown
2024-08-03 12:35:32 +02:00
if o . fs . opt . Decompress && deref ( o . contentEncoding ) == "gzip" {
2022-07-29 18:01:59 +02:00
o . bytes = - 1
o . md5 = ""
}
2013-01-08 19:53:35 +01:00
}
// ModTime returns the modification time of the object
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
2019-06-17 10:34:30 +02:00
func ( o * Object ) ModTime ( ctx context . Context ) time . Time {
2020-11-05 12:33:32 +01:00
if o . fs . ci . UseServerModTime {
2018-04-13 14:32:17 +02:00
return o . lastModified
}
2019-06-17 10:34:30 +02:00
err := o . readMetaData ( ctx )
2013-01-08 19:53:35 +01:00
if err != nil {
2017-02-09 12:01:20 +01:00
fs . Logf ( o , "Failed to read metadata: %v" , err )
2013-01-08 19:53:35 +01:00
return time . Now ( )
}
// read mtime out of metadata if available
2013-06-27 21:13:07 +02:00
d , ok := o . meta [ metaMtime ]
2022-06-22 16:40:30 +02:00
if ! ok {
2017-02-09 12:01:20 +01:00
// fs.Debugf(o, "No metadata")
2013-06-27 21:13:07 +02:00
return o . lastModified
2013-01-08 19:53:35 +01:00
}
2022-06-22 16:40:30 +02:00
modTime , err := swift . FloatStringToTime ( d )
2013-01-08 19:53:35 +01:00
if err != nil {
2017-02-09 12:01:20 +01:00
fs . Logf ( o , "Failed to read mtime from object: %v" , err )
2013-06-27 21:13:07 +02:00
return o . lastModified
2013-01-08 19:53:35 +01:00
}
return modTime
}
2015-09-22 19:47:16 +02:00
// SetModTime sets the modification time of the local fs object
2019-06-17 10:34:30 +02:00
func ( o * Object ) SetModTime ( ctx context . Context , modTime time . Time ) error {
err := o . readMetaData ( ctx )
2013-01-08 23:31:16 +01:00
if err != nil {
2016-03-22 16:07:10 +01:00
return err
2013-01-08 23:31:16 +01:00
}
2022-06-22 16:40:30 +02:00
o . meta [ metaMtime ] = swift . TimeToFloatString ( modTime )
2014-12-23 13:09:02 +01:00
2019-09-09 21:44:50 +02:00
// Can't update metadata here, so return this error to force a recopy
2022-05-24 13:32:39 +02:00
if o . storageClass != nil && ( * o . storageClass == "GLACIER" || * o . storageClass == "DEEP_ARCHIVE" ) {
2019-09-09 21:44:50 +02:00
return fs . ErrorCantSetModTime
}
2016-01-02 09:58:48 +01:00
2014-12-23 13:09:02 +01:00
// Copy the object to itself to update the metadata
2019-08-09 12:29:36 +02:00
bucket , bucketPath := o . split ( )
2014-12-23 13:09:02 +01:00
req := s3 . CopyObjectInput {
2019-09-09 21:44:50 +02:00
ContentType : aws . String ( fs . MimeType ( ctx , o ) ) , // Guess the content type
2022-06-22 16:40:30 +02:00
Metadata : mapToS3Metadata ( o . meta ) ,
2024-08-03 12:35:32 +02:00
MetadataDirective : types . MetadataDirectiveReplace , // replace metadata with that passed in
2019-06-03 16:28:19 +02:00
}
2020-12-03 03:30:06 +01:00
if o . fs . opt . RequesterPays {
2024-08-03 12:35:32 +02:00
req . RequestPayer = types . RequestPayerRequester
2020-12-03 03:30:06 +01:00
}
2020-07-30 11:52:32 +02:00
return o . fs . copy ( ctx , & req , bucket , bucketPath , bucket , bucketPath , o )
2013-01-08 19:53:35 +01:00
}
2015-09-22 19:47:16 +02:00
// Storable raturns a boolean indicating if this object is storable
2015-11-07 12:14:46 +01:00
func ( o * Object ) Storable ( ) bool {
2013-01-08 19:53:35 +01:00
return true
}
2021-10-14 12:19:38 +02:00
func ( o * Object ) downloadFromURL ( ctx context . Context , bucketPath string , options ... fs . OpenOption ) ( in io . ReadCloser , err error ) {
url := o . fs . opt . DownloadURL + bucketPath
var resp * http . Response
opts := rest . Opts {
Method : "GET" ,
RootURL : url ,
Options : options ,
}
err = o . fs . pacer . Call ( func ( ) ( bool , error ) {
resp , err = o . fs . srvRest . Call ( ctx , & opts )
return o . fs . shouldRetry ( ctx , err )
} )
if err != nil {
return nil , err
}
2022-03-20 11:12:48 +01:00
contentLength := rest . ParseSizeFromHeaders ( resp . Header )
if contentLength < 0 {
fs . Debugf ( o , "Failed to parse file size from headers" )
2021-10-14 12:19:38 +02:00
}
2022-03-20 11:12:48 +01:00
lastModified , err := http . ParseTime ( resp . Header . Get ( "Last-Modified" ) )
2021-10-14 12:19:38 +02:00
if err != nil {
fs . Debugf ( o , "Failed to parse last modified from string %s, %v" , resp . Header . Get ( "Last-Modified" ) , err )
}
2024-08-03 12:35:32 +02:00
metaData := make ( map [ string ] string )
2021-10-14 12:19:38 +02:00
for key , value := range resp . Header {
2022-05-24 13:32:39 +02:00
key = strings . ToLower ( key )
if strings . HasPrefix ( key , "x-amz-meta-" ) {
2021-10-14 12:19:38 +02:00
metaKey := strings . TrimPrefix ( key , "x-amz-meta-" )
2024-08-03 12:35:32 +02:00
metaData [ metaKey ] = value [ 0 ]
2021-10-14 12:19:38 +02:00
}
}
2022-05-24 13:32:39 +02:00
header := func ( k string ) * string {
v := resp . Header . Get ( k )
if v == "" {
return nil
}
return & v
}
var head = s3 . HeadObjectOutput {
ETag : header ( "Etag" ) ,
2022-03-20 11:12:48 +01:00
ContentLength : & contentLength ,
2022-05-24 13:32:39 +02:00
LastModified : & lastModified ,
Metadata : metaData ,
CacheControl : header ( "Cache-Control" ) ,
ContentDisposition : header ( "Content-Disposition" ) ,
ContentEncoding : header ( "Content-Encoding" ) ,
ContentLanguage : header ( "Content-Language" ) ,
ContentType : header ( "Content-Type" ) ,
2024-08-03 12:35:32 +02:00
StorageClass : types . StorageClass ( * header ( "X-Amz-Storage-Class" ) ) ,
2022-05-24 13:32:39 +02:00
}
o . setMetaData ( & head )
2021-10-14 12:19:38 +02:00
return resp . Body , err
}
2013-01-08 19:53:35 +01:00
// Open an object for read
2019-06-17 10:34:30 +02:00
func ( o * Object ) Open ( ctx context . Context , options ... fs . OpenOption ) ( in io . ReadCloser , err error ) {
2019-08-09 12:29:36 +02:00
bucket , bucketPath := o . split ( )
2021-10-14 12:19:38 +02:00
if o . fs . opt . DownloadURL != "" {
return o . downloadFromURL ( ctx , bucketPath , options ... )
}
2014-12-23 13:09:02 +01:00
req := s3 . GetObjectInput {
2022-07-25 17:06:15 +02:00
Bucket : & bucket ,
Key : & bucketPath ,
VersionId : o . versionID ,
2014-12-23 13:09:02 +01:00
}
2020-12-03 03:30:06 +01:00
if o . fs . opt . RequesterPays {
2024-08-03 12:35:32 +02:00
req . RequestPayer = types . RequestPayerRequester
2020-12-03 03:30:06 +01:00
}
2020-03-30 12:26:52 +02:00
if o . fs . opt . SSECustomerAlgorithm != "" {
req . SSECustomerAlgorithm = & o . fs . opt . SSECustomerAlgorithm
}
2024-08-07 11:14:58 +02:00
if o . fs . opt . SSECustomerKeyBase64 != "" {
req . SSECustomerKey = & o . fs . opt . SSECustomerKeyBase64
2020-03-30 12:26:52 +02:00
}
if o . fs . opt . SSECustomerKeyMD5 != "" {
req . SSECustomerKeyMD5 = & o . fs . opt . SSECustomerKeyMD5
}
2024-08-03 12:35:32 +02:00
// httpReq, err := s3.NewPresignClient(o.fs.c).PresignGetObject(ctx, &req)
// if err != nil {
// return nil, err
// }
2019-08-06 16:18:08 +02:00
fs . FixRangeOption ( options , o . bytes )
2022-07-29 18:01:59 +02:00
2024-08-03 12:35:32 +02:00
var APIOptions [ ] func ( * middleware . Stack ) error
2022-07-29 18:01:59 +02:00
// Override the automatic decompression in the transport to
// download compressed files as-is
2023-01-09 07:54:51 +01:00
if o . fs . opt . UseAcceptEncodingGzip . Value {
2024-08-03 12:35:32 +02:00
APIOptions = append ( APIOptions , smithyhttp . AddHeaderValue ( "Accept-Encoding" , "gzip" ) )
2023-01-09 07:54:51 +01:00
}
2022-07-29 18:01:59 +02:00
2016-09-10 12:29:57 +02:00
for _ , option := range options {
switch option . ( type ) {
case * fs . RangeOption , * fs . SeekOption :
_ , value := option . Header ( )
req . Range = & value
2020-02-10 10:02:04 +01:00
case * fs . HTTPOption :
key , value := option . Header ( )
2024-08-03 12:35:32 +02:00
APIOptions = append ( APIOptions , smithyhttp . AddHeaderValue ( key , value ) )
2016-09-10 12:29:57 +02:00
default :
if option . Mandatory ( ) {
2017-02-09 12:01:20 +01:00
fs . Logf ( o , "Unsupported mandatory option: %v" , option )
2016-09-10 12:29:57 +02:00
}
}
}
2024-08-03 12:35:32 +02:00
var resp * s3 . GetObjectOutput
2018-09-03 06:41:04 +02:00
err = o . fs . pacer . Call ( func ( ) ( bool , error ) {
var err error
2024-08-03 12:35:32 +02:00
resp , err = o . fs . c . GetObject ( ctx , & req , s3 . WithAPIOptions ( APIOptions ... ) )
2021-03-16 16:50:02 +01:00
return o . fs . shouldRetry ( ctx , err )
2018-09-03 06:41:04 +02:00
} )
2024-08-03 12:35:32 +02:00
var awsError smithy . APIError
if errors . As ( err , & awsError ) {
if awsError . ErrorCode ( ) == "InvalidObjectState" {
2021-11-04 11:12:57 +01:00
return nil , fmt . Errorf ( "Object in GLACIER, restore first: bucket=%q, key=%q" , bucket , bucketPath )
2017-09-09 14:02:26 +02:00
}
}
2014-12-23 13:09:02 +01:00
if err != nil {
return nil , err
}
2022-02-16 18:50:11 +01:00
2021-04-28 12:05:54 +02:00
// read size from ContentLength or ContentRange
size := resp . ContentLength
if resp . ContentRange != nil {
var contentRange = * resp . ContentRange
slash := strings . IndexRune ( contentRange , '/' )
if slash >= 0 {
i , err := strconv . ParseInt ( contentRange [ slash + 1 : ] , 10 , 64 )
if err == nil {
size = & i
} else {
fs . Debugf ( o , "Failed to find parse integer from in %q: %v" , contentRange , err )
}
} else {
fs . Debugf ( o , "Failed to find length in %q" , contentRange )
}
}
2022-05-24 13:32:39 +02:00
var head s3 . HeadObjectOutput
2022-07-28 12:49:19 +02:00
//structs.SetFrom(&head, resp)
setFrom_s3HeadObjectOutput_s3GetObjectOutput ( & head , resp )
2022-05-24 13:32:39 +02:00
head . ContentLength = size
o . setMetaData ( & head )
2022-07-29 18:01:59 +02:00
// Decompress body if necessary
2024-08-03 12:35:32 +02:00
if deref ( resp . ContentEncoding ) == "gzip" {
2022-10-29 00:23:29 +02:00
if o . fs . opt . Decompress || ( resp . ContentLength == nil && o . fs . opt . MightGzip . Value ) {
2022-07-29 18:01:59 +02:00
return readers . NewGzipReader ( resp . Body )
}
o . fs . warnCompressed . Do ( func ( ) {
fs . Logf ( o , "Not decompressing 'Content-Encoding: gzip' compressed file. Use --s3-decompress to override" )
} )
}
2014-12-23 13:09:02 +01:00
return resp . Body , nil
2013-01-08 19:53:35 +01:00
}
2019-11-06 11:41:03 +01:00
var warnStreamUpload sync . Once
2023-08-15 21:38:02 +02:00
// state of ChunkWriter
type s3ChunkWriter struct {
chunkSize int64
size int64
f * Fs
bucket * string
key * string
2023-08-29 13:57:33 +02:00
uploadID * string
2023-08-15 21:38:02 +02:00
multiPartUploadInput * s3 . CreateMultipartUploadInput
completedPartsMu sync . Mutex
2024-08-03 12:35:32 +02:00
completedParts [ ] types . CompletedPart
2023-08-15 21:38:02 +02:00
eTag string
versionID string
md5sMu sync . Mutex
md5s [ ] byte
ui uploadInfo
o * Object
}
2023-08-15 15:27:21 +02:00
// OpenChunkWriter returns the chunk size and a ChunkWriter
//
// Pass in the remote and the src object
// You can also use options to hint at the desired chunk size
2023-09-01 18:25:15 +02:00
func ( f * Fs ) OpenChunkWriter ( ctx context . Context , remote string , src fs . ObjectInfo , options ... fs . OpenOption ) ( info fs . ChunkWriterInfo , writer fs . ChunkWriter , err error ) {
2023-07-18 20:37:31 +02:00
// Temporary Object under construction
o := & Object {
fs : f ,
remote : remote ,
2019-12-31 00:17:06 +01:00
}
2024-03-05 18:21:06 +01:00
ui , err := o . prepareUpload ( ctx , src , options , false )
2023-07-18 20:37:31 +02:00
if err != nil {
2023-09-01 18:25:15 +02:00
return info , nil , fmt . Errorf ( "failed to prepare upload: %w" , err )
2023-07-18 20:37:31 +02:00
}
//structs.SetFrom(&mReq, req)
var mReq s3 . CreateMultipartUploadInput
2023-08-15 21:38:02 +02:00
setFrom_s3CreateMultipartUploadInput_s3PutObjectInput ( & mReq , ui . req )
2019-12-31 00:17:06 +01:00
2020-06-08 19:22:34 +02:00
uploadParts := f . opt . MaxUploadParts
if uploadParts < 1 {
uploadParts = 1
} else if uploadParts > maxUploadParts {
uploadParts = maxUploadParts
}
2023-07-18 20:37:31 +02:00
size := src . Size ( )
2020-06-08 19:22:34 +02:00
2019-12-31 00:17:06 +01:00
// calculate size of parts
2023-07-18 20:37:31 +02:00
chunkSize := f . opt . ChunkSize
2019-12-31 00:17:06 +01:00
2021-03-02 20:11:57 +01:00
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
// 48 GiB which seems like a not too unreasonable limit.
2019-12-31 00:17:06 +01:00
if size == - 1 {
warnStreamUpload . Do ( func ( ) {
fs . Logf ( f , "Streaming uploads using chunk size %v will have maximum file size of %v" ,
2023-07-18 20:37:31 +02:00
f . opt . ChunkSize , fs . SizeSuffix ( int64 ( chunkSize ) * int64 ( uploadParts ) ) )
2019-12-31 00:17:06 +01:00
} )
} else {
2023-07-18 20:37:31 +02:00
chunkSize = chunksize . Calculator ( src , size , uploadParts , chunkSize )
2019-12-31 00:17:06 +01:00
}
2023-08-15 15:27:21 +02:00
var mOut * s3 . CreateMultipartUploadOutput
err = f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
mOut , err = f . c . CreateMultipartUpload ( ctx , & mReq )
2023-12-16 12:34:38 +01:00
if err == nil {
if mOut == nil {
err = fserrors . RetryErrorf ( "internal error: no info from multipart upload" )
} else if mOut . UploadId == nil {
err = fserrors . RetryErrorf ( "internal error: no UploadId in multpart upload: %#v" , * mOut )
}
}
2023-08-15 15:27:21 +02:00
return f . shouldRetry ( ctx , err )
} )
2023-07-18 20:37:31 +02:00
if err != nil {
2023-09-01 18:25:15 +02:00
return info , nil , fmt . Errorf ( "create multipart upload failed: %w" , err )
2023-07-18 20:37:31 +02:00
}
chunkWriter := & s3ChunkWriter {
chunkSize : int64 ( chunkSize ) ,
2023-08-15 15:27:21 +02:00
size : size ,
2023-07-18 20:37:31 +02:00
f : f ,
bucket : mOut . Bucket ,
key : mOut . Key ,
2023-08-29 13:57:33 +02:00
uploadID : mOut . UploadId ,
2023-07-18 20:37:31 +02:00
multiPartUploadInput : & mReq ,
2024-08-03 12:35:32 +02:00
completedParts : make ( [ ] types . CompletedPart , 0 ) ,
2023-08-15 21:38:02 +02:00
ui : ui ,
o : o ,
2023-07-18 20:37:31 +02:00
}
2023-09-01 18:25:15 +02:00
info = fs . ChunkWriterInfo {
ChunkSize : int64 ( chunkSize ) ,
Concurrency : o . fs . opt . UploadConcurrency ,
LeavePartsOnError : o . fs . opt . LeavePartsOnError ,
}
2023-08-15 21:38:02 +02:00
fs . Debugf ( o , "open chunk writer: started multipart upload: %v" , * mOut . UploadId )
2023-09-01 18:25:15 +02:00
return info , chunkWriter , err
2023-07-18 20:37:31 +02:00
}
2020-02-19 11:17:25 +01:00
2023-08-15 15:27:21 +02:00
// add a part number and etag to the completed parts
2024-08-03 12:35:32 +02:00
func ( w * s3ChunkWriter ) addCompletedPart ( partNum * int32 , eTag * string ) {
2023-08-15 15:27:21 +02:00
w . completedPartsMu . Lock ( )
defer w . completedPartsMu . Unlock ( )
2024-08-03 12:35:32 +02:00
w . completedParts = append ( w . completedParts , types . CompletedPart {
2023-08-15 15:27:21 +02:00
PartNumber : partNum ,
ETag : eTag ,
} )
}
// addMd5 adds a binary md5 to the md5 calculated so far
func ( w * s3ChunkWriter ) addMd5 ( md5binary * [ ] byte , chunkNumber int64 ) {
w . md5sMu . Lock ( )
defer w . md5sMu . Unlock ( )
start := chunkNumber * md5 . Size
end := start + md5 . Size
if extend := end - int64 ( len ( w . md5s ) ) ; extend > 0 {
w . md5s = append ( w . md5s , make ( [ ] byte , extend ) ... )
}
2024-05-31 14:33:10 +02:00
copy ( w . md5s [ start : end ] , ( * md5binary ) )
2023-08-15 15:27:21 +02:00
}
// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
2023-08-19 18:30:55 +02:00
func ( w * s3ChunkWriter ) WriteChunk ( ctx context . Context , chunkNumber int , reader io . ReadSeeker ) ( int64 , error ) {
2023-07-18 20:37:31 +02:00
if chunkNumber < 0 {
err := fmt . Errorf ( "invalid chunk number provided: %v" , chunkNumber )
return - 1 , err
}
2023-08-24 18:15:18 +02:00
// Only account after the checksum reads have been done
if do , ok := reader . ( pool . DelayAccountinger ) ; ok {
// To figure out this number, do a transfer and if the accounted size is 0 or a
// multiple of what it should be, increase or decrease this number.
do . DelayAccounting ( 3 )
}
2023-07-18 20:37:31 +02:00
// create checksum of buffer for integrity checking
// currently there is no way to calculate the md5 without reading the chunk a 2nd time (1st read is in uploadMultipart)
// possible in AWS SDK v2 with trailers?
m := md5 . New ( )
currentChunkSize , err := io . Copy ( m , reader )
2023-08-15 21:38:02 +02:00
if err != nil {
2023-07-18 20:37:31 +02:00
return - 1 , err
}
2023-09-03 13:37:20 +02:00
// If no data read and not the first chunk, don't write the chunk
if currentChunkSize == 0 && chunkNumber != 0 {
2023-08-15 21:38:02 +02:00
return 0 , nil
}
2023-07-18 20:37:31 +02:00
md5sumBinary := m . Sum ( [ ] byte { } )
2023-08-15 15:27:21 +02:00
w . addMd5 ( & md5sumBinary , int64 ( chunkNumber ) )
2024-05-31 14:33:10 +02:00
md5sum := base64 . StdEncoding . EncodeToString ( md5sumBinary )
2023-07-18 20:37:31 +02:00
// S3 requires 1 <= PartNumber <= 10000
2024-08-03 12:35:32 +02:00
s3PartNumber := aws . Int32 ( int32 ( chunkNumber + 1 ) )
2023-07-18 20:37:31 +02:00
uploadPartReq := & s3 . UploadPartInput {
Body : reader ,
Bucket : w . bucket ,
Key : w . key ,
PartNumber : s3PartNumber ,
2023-08-29 13:57:33 +02:00
UploadId : w . uploadID ,
2023-07-18 20:37:31 +02:00
ContentMD5 : & md5sum ,
ContentLength : aws . Int64 ( currentChunkSize ) ,
RequestPayer : w . multiPartUploadInput . RequestPayer ,
SSECustomerAlgorithm : w . multiPartUploadInput . SSECustomerAlgorithm ,
SSECustomerKey : w . multiPartUploadInput . SSECustomerKey ,
SSECustomerKeyMD5 : w . multiPartUploadInput . SSECustomerKeyMD5 ,
}
2023-08-15 15:27:21 +02:00
var uout * s3 . UploadPartOutput
err = w . f . pacer . Call ( func ( ) ( bool , error ) {
2023-08-15 21:38:02 +02:00
// rewind the reader on retry and after reading md5
_ , err = reader . Seek ( 0 , io . SeekStart )
if err != nil {
return false , err
}
2024-08-03 12:35:32 +02:00
uout , err = w . f . c . UploadPart ( ctx , uploadPartReq )
2023-08-15 15:27:21 +02:00
if err != nil {
if chunkNumber <= 8 {
2023-08-15 21:38:02 +02:00
return w . f . shouldRetry ( ctx , err )
2023-08-15 15:27:21 +02:00
}
// retry all chunks once have done the first few
return true , err
}
return false , nil
} )
2023-07-18 20:37:31 +02:00
if err != nil {
2023-08-15 15:27:21 +02:00
return - 1 , fmt . Errorf ( "failed to upload chunk %d with %v bytes: %w" , chunkNumber + 1 , currentChunkSize , err )
2023-07-18 20:37:31 +02:00
}
2023-08-15 15:27:21 +02:00
w . addCompletedPart ( s3PartNumber , uout . ETag )
2023-07-18 20:37:31 +02:00
2023-08-15 21:38:02 +02:00
fs . Debugf ( w . o , "multipart upload wrote chunk %d with %v bytes and etag %v" , chunkNumber + 1 , currentChunkSize , * uout . ETag )
2023-07-18 20:37:31 +02:00
return currentChunkSize , err
}
2023-09-23 13:20:01 +02:00
// Abort the multipart upload
2023-08-19 18:30:55 +02:00
func ( w * s3ChunkWriter ) Abort ( ctx context . Context ) error {
2023-08-15 15:27:21 +02:00
err := w . f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
_ , err := w . f . c . AbortMultipartUpload ( context . Background ( ) , & s3 . AbortMultipartUploadInput {
2023-08-15 15:27:21 +02:00
Bucket : w . bucket ,
Key : w . key ,
2023-08-29 13:57:33 +02:00
UploadId : w . uploadID ,
2023-08-15 15:27:21 +02:00
RequestPayer : w . multiPartUploadInput . RequestPayer ,
} )
2023-08-15 21:38:02 +02:00
return w . f . shouldRetry ( ctx , err )
2023-07-18 20:37:31 +02:00
} )
if err != nil {
2023-08-29 13:57:33 +02:00
return fmt . Errorf ( "failed to abort multipart upload %q: %w" , * w . uploadID , err )
2023-07-18 20:37:31 +02:00
}
2023-08-29 13:57:33 +02:00
fs . Debugf ( w . o , "multipart upload %q aborted" , * w . uploadID )
2023-07-18 20:37:31 +02:00
return err
}
2023-08-15 15:27:21 +02:00
// Close and finalise the multipart upload
2023-08-19 18:30:55 +02:00
func ( w * s3ChunkWriter ) Close ( ctx context . Context ) ( err error ) {
2023-07-18 20:37:31 +02:00
// sort the completed parts by part number
sort . Slice ( w . completedParts , func ( i , j int ) bool {
return * w . completedParts [ i ] . PartNumber < * w . completedParts [ j ] . PartNumber
} )
2023-08-15 15:27:21 +02:00
var resp * s3 . CompleteMultipartUploadOutput
err = w . f . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
resp , err = w . f . c . CompleteMultipartUpload ( ctx , & s3 . CompleteMultipartUploadInput {
2023-08-15 15:27:21 +02:00
Bucket : w . bucket ,
Key : w . key ,
2024-08-03 12:35:32 +02:00
MultipartUpload : & types . CompletedMultipartUpload {
2023-08-15 15:27:21 +02:00
Parts : w . completedParts ,
} ,
RequestPayer : w . multiPartUploadInput . RequestPayer ,
2023-08-29 13:57:33 +02:00
UploadId : w . uploadID ,
2023-08-15 15:27:21 +02:00
} )
2023-08-15 21:38:02 +02:00
return w . f . shouldRetry ( ctx , err )
2023-07-18 20:37:31 +02:00
} )
if err != nil {
2023-08-29 13:57:33 +02:00
return fmt . Errorf ( "failed to complete multipart upload %q: %w" , * w . uploadID , err )
2023-07-18 20:37:31 +02:00
}
if resp != nil {
if resp . ETag != nil {
w . eTag = * resp . ETag
}
if resp . VersionId != nil {
w . versionID = * resp . VersionId
}
}
2023-08-29 13:57:33 +02:00
fs . Debugf ( w . o , "multipart upload %q finished" , * w . uploadID )
2023-07-18 20:37:31 +02:00
return err
}
2023-08-15 21:38:02 +02:00
func ( o * Object ) uploadMultipart ( ctx context . Context , src fs . ObjectInfo , in io . Reader , options ... fs . OpenOption ) ( wantETag , gotETag string , versionID * string , ui uploadInfo , err error ) {
chunkWriter , err := multipart . UploadMultipart ( ctx , src , in , multipart . UploadMultipartOptions {
2023-09-01 18:25:15 +02:00
Open : o . fs ,
OpenOptions : options ,
2023-08-15 21:38:02 +02:00
} )
2019-12-31 00:17:06 +01:00
if err != nil {
2023-08-15 21:38:02 +02:00
return wantETag , gotETag , versionID , ui , err
2019-12-31 00:17:06 +01:00
}
2023-07-18 20:37:31 +02:00
var s3cw * s3ChunkWriter = chunkWriter . ( * s3ChunkWriter )
gotETag = s3cw . eTag
versionID = aws . String ( s3cw . versionID )
hashOfHashes := md5 . Sum ( s3cw . md5s )
wantETag = fmt . Sprintf ( "%s-%d" , hex . EncodeToString ( hashOfHashes [ : ] ) , len ( s3cw . completedParts ) )
2023-08-15 21:38:02 +02:00
return wantETag , gotETag , versionID , s3cw . ui , nil
2019-12-31 00:17:06 +01:00
}
2022-05-03 18:39:01 +02:00
// Upload a single part using PutObject
2022-07-25 17:06:15 +02:00
func ( o * Object ) uploadSinglepartPutObject ( ctx context . Context , req * s3 . PutObjectInput , size int64 , in io . Reader ) ( etag string , lastModified time . Time , versionID * string , err error ) {
2024-08-03 12:35:32 +02:00
req . Body = io . NopCloser ( in )
var options = [ ] func ( * s3 . Options ) { }
if o . fs . opt . UseUnsignedPayload . Value {
options = append ( options , s3 . WithAPIOptions (
// avoids operation error S3: PutObject, failed to compute payload hash: failed to seek body to start, request stream is not seekable
v4signer . SwapComputePayloadSHA256ForUnsignedPayloadMiddleware ,
) )
}
// Can't retry single part uploads as only have an io.Reader
options = append ( options , func ( s3opt * s3 . Options ) {
s3opt . RetryMaxAttempts = 1
} )
var resp * s3 . PutObjectOutput
2022-05-03 18:39:01 +02:00
err = o . fs . pacer . CallNoRetry ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
resp , err = o . fs . c . PutObject ( ctx , req , options ... )
2022-05-03 18:39:01 +02:00
return o . fs . shouldRetry ( ctx , err )
} )
if err != nil {
2022-07-25 17:06:15 +02:00
return etag , lastModified , nil , err
2022-05-03 18:39:01 +02:00
}
lastModified = time . Now ( )
2022-07-25 17:06:15 +02:00
if resp != nil {
2024-08-03 12:35:32 +02:00
etag = deref ( resp . ETag )
2022-07-25 17:06:15 +02:00
versionID = resp . VersionId
}
return etag , lastModified , versionID , nil
2022-05-03 18:39:01 +02:00
}
// Upload a single part using a presigned request
2022-07-25 17:06:15 +02:00
func ( o * Object ) uploadSinglepartPresignedRequest ( ctx context . Context , req * s3 . PutObjectInput , size int64 , in io . Reader ) ( etag string , lastModified time . Time , versionID * string , err error ) {
2024-08-03 12:35:32 +02:00
// Create the presigned request
putReq , err := s3 . NewPresignClient ( o . fs . c ) . PresignPutObject ( ctx , req , s3 . WithPresignExpires ( 15 * time . Minute ) )
2022-05-03 18:39:01 +02:00
if err != nil {
2022-07-25 17:06:15 +02:00
return etag , lastModified , nil , fmt . Errorf ( "s3 upload: sign request: %w" , err )
2022-05-03 18:39:01 +02:00
}
// Set request to nil if empty so as not to make chunked encoding
if size == 0 {
in = nil
}
// create the vanilla http request
2024-08-03 12:35:32 +02:00
httpReq , err := http . NewRequestWithContext ( ctx , "PUT" , putReq . URL , in )
2022-05-03 18:39:01 +02:00
if err != nil {
2022-07-25 17:06:15 +02:00
return etag , lastModified , nil , fmt . Errorf ( "s3 upload: new request: %w" , err )
2022-05-03 18:39:01 +02:00
}
// set the headers we signed and the length
2024-08-03 12:35:32 +02:00
httpReq . Header = putReq . SignedHeader
2022-05-03 18:39:01 +02:00
httpReq . ContentLength = size
var resp * http . Response
err = o . fs . pacer . CallNoRetry ( func ( ) ( bool , error ) {
var err error
resp , err = o . fs . srv . Do ( httpReq )
if err != nil {
return o . fs . shouldRetry ( ctx , err )
}
body , err := rest . ReadBody ( resp )
if err != nil {
return o . fs . shouldRetry ( ctx , err )
}
if resp . StatusCode >= 200 && resp . StatusCode < 299 {
return false , nil
}
err = fmt . Errorf ( "s3 upload: %s: %s" , resp . Status , body )
return fserrors . ShouldRetryHTTP ( resp , retryErrorCodes ) , err
} )
if err != nil {
2022-07-25 17:06:15 +02:00
return etag , lastModified , nil , err
2022-05-03 18:39:01 +02:00
}
if resp != nil {
if date , err := http . ParseTime ( resp . Header . Get ( "Date" ) ) ; err != nil {
lastModified = date
}
etag = resp . Header . Get ( "Etag" )
2022-07-25 17:06:15 +02:00
vID := resp . Header . Get ( "x-amz-version-id" )
if vID != "" {
versionID = & vID
}
2022-05-03 18:39:01 +02:00
}
2022-07-25 17:06:15 +02:00
return etag , lastModified , versionID , nil
2022-05-03 18:39:01 +02:00
}
2023-08-15 21:38:02 +02:00
// Info needed for an upload
type uploadInfo struct {
req * s3 . PutObjectInput
md5sumHex string
}
// Prepare object for being uploaded
2024-03-05 18:21:06 +01:00
//
// If noHash is true the md5sum will not be calculated
func ( o * Object ) prepareUpload ( ctx context . Context , src fs . ObjectInfo , options [ ] fs . OpenOption , noHash bool ) ( ui uploadInfo , err error ) {
2019-08-09 12:29:36 +02:00
bucket , bucketPath := o . split ( )
2023-04-26 11:59:17 +02:00
// Create parent dir/bucket if not saving directory marker
if ! strings . HasSuffix ( o . remote , "/" ) {
err := o . fs . mkdirParent ( ctx , o . remote )
if err != nil {
2023-08-15 21:38:02 +02:00
return ui , err
2023-04-26 11:59:17 +02:00
}
2017-06-07 15:16:50 +02:00
}
2019-06-17 10:34:30 +02:00
modTime := src . ModTime ( ctx )
2014-04-18 18:04:21 +02:00
2023-08-15 21:38:02 +02:00
ui . req = & s3 . PutObjectInput {
2022-05-24 13:32:39 +02:00
Bucket : & bucket ,
2024-08-03 12:35:32 +02:00
ACL : types . ObjectCannedACL ( o . fs . opt . ACL ) ,
2022-05-24 13:32:39 +02:00
Key : & bucketPath ,
}
// Fetch metadata if --metadata is in use
2023-10-24 00:47:18 +02:00
meta , err := fs . GetMetadataOptions ( ctx , o . fs , src , options )
2022-05-24 13:32:39 +02:00
if err != nil {
2023-08-15 21:38:02 +02:00
return ui , fmt . Errorf ( "failed to read metadata from source object: %w" , err )
2022-05-24 13:32:39 +02:00
}
2024-08-03 12:35:32 +02:00
ui . req . Metadata = make ( map [ string ] string , len ( meta ) + 2 )
2022-05-24 13:32:39 +02:00
// merge metadata into request and user metadata
for k , v := range meta {
pv := aws . String ( v )
k = strings . ToLower ( k )
2022-10-12 09:55:58 +02:00
if o . fs . opt . NoSystemMetadata {
2024-08-03 12:35:32 +02:00
ui . req . Metadata [ k ] = v
2022-10-12 09:55:58 +02:00
continue
}
2022-05-24 13:32:39 +02:00
switch k {
case "cache-control" :
2023-08-15 21:38:02 +02:00
ui . req . CacheControl = pv
2022-05-24 13:32:39 +02:00
case "content-disposition" :
2023-08-15 21:38:02 +02:00
ui . req . ContentDisposition = pv
2022-05-24 13:32:39 +02:00
case "content-encoding" :
2023-08-15 21:38:02 +02:00
ui . req . ContentEncoding = pv
2022-05-24 13:32:39 +02:00
case "content-language" :
2023-08-15 21:38:02 +02:00
ui . req . ContentLanguage = pv
2022-05-24 13:32:39 +02:00
case "content-type" :
2023-08-15 21:38:02 +02:00
ui . req . ContentType = pv
2022-05-24 13:32:39 +02:00
case "x-amz-tagging" :
2023-08-15 21:38:02 +02:00
ui . req . Tagging = pv
2022-05-24 13:32:39 +02:00
case "tier" :
// ignore
case "mtime" :
// mtime in meta overrides source ModTime
metaModTime , err := time . Parse ( time . RFC3339Nano , v )
if err != nil {
fs . Debugf ( o , "failed to parse metadata %s: %q: %v" , k , v , err )
} else {
modTime = metaModTime
}
case "btime" :
// write as metadata since we can't set it
2024-08-03 12:35:32 +02:00
ui . req . Metadata [ k ] = v
2022-05-24 13:32:39 +02:00
default :
2024-08-03 12:35:32 +02:00
ui . req . Metadata [ k ] = v
2022-05-24 13:32:39 +02:00
}
2014-12-23 13:09:02 +01:00
}
2022-05-24 13:32:39 +02:00
// Set the mtime in the meta data
2024-08-03 12:35:32 +02:00
ui . req . Metadata [ metaMtime ] = swift . TimeToFloatString ( modTime )
2022-05-24 13:32:39 +02:00
2020-01-07 20:23:08 +01:00
// read the md5sum if available
Spelling fixes
Fix spelling of: above, already, anonymous, associated,
authentication, bandwidth, because, between, blocks, calculate,
candidates, cautious, changelog, cleaner, clipboard, command,
completely, concurrently, considered, constructs, corrupt, current,
daemon, dependencies, deprecated, directory, dispatcher, download,
eligible, ellipsis, encrypter, endpoint, entrieslist, essentially,
existing writers, existing, expires, filesystem, flushing, frequently,
hierarchy, however, implementation, implements, inaccurate,
individually, insensitive, longer, maximum, metadata, modified,
multipart, namedirfirst, nextcloud, obscured, opened, optional,
owncloud, pacific, passphrase, password, permanently, persimmon,
positive, potato, protocol, quota, receiving, recommends, referring,
requires, revisited, satisfied, satisfies, satisfy, semver,
serialized, session, storage, strategies, stringlist, successful,
supported, surprise, temporarily, temporary, transactions, unneeded,
update, uploads, wrapped
Signed-off-by: Josh Soref <jsoref@users.noreply.github.com>
2020-10-09 02:17:24 +02:00
// - for non multipart
2020-01-07 20:23:08 +01:00
// - so we can add a ContentMD5
2020-11-23 12:53:31 +01:00
// - so we can add the md5sum in the metadata as metaMD5Hash if using SSE/SSE-C
2020-01-07 20:23:08 +01:00
// - for multipart provided checksums aren't disabled
// - so we can add the md5sum in the metadata as metaMD5Hash
2022-01-28 17:18:40 +01:00
var md5sumBase64 string
2023-07-18 20:37:31 +02:00
size := src . Size ( )
multipart := size < 0 || size >= int64 ( o . fs . opt . UploadCutoff )
2024-03-05 18:21:06 +01:00
if ! noHash && ( ! multipart || ! o . fs . opt . DisableChecksum ) {
2023-08-15 21:38:02 +02:00
ui . md5sumHex , err = src . Hash ( ctx , hash . MD5 )
if err == nil && matchMd5 . MatchString ( ui . md5sumHex ) {
hashBytes , err := hex . DecodeString ( ui . md5sumHex )
2018-01-06 15:30:10 +01:00
if err == nil {
2022-01-28 17:18:40 +01:00
md5sumBase64 = base64 . StdEncoding . EncodeToString ( hashBytes )
2020-11-23 12:53:31 +01:00
if ( multipart || o . fs . etagIsNotMD5 ) && ! o . fs . opt . DisableChecksum {
// Set the md5sum as metadata on the object if
// - a multipart upload
// - the Etag is not an MD5, eg when using SSE/SSE-C
// provided checksums aren't disabled
2024-08-03 12:35:32 +02:00
ui . req . Metadata [ metaMD5Hash ] = md5sumBase64
2018-11-26 22:09:23 +01:00
}
2018-01-06 15:30:10 +01:00
}
}
}
2023-07-18 20:37:31 +02:00
// Set the content type if it isn't set already
2023-08-15 21:38:02 +02:00
if ui . req . ContentType == nil {
ui . req . ContentType = aws . String ( fs . MimeType ( ctx , src ) )
2019-12-31 00:17:06 +01:00
}
2022-05-03 18:39:01 +02:00
if size >= 0 {
2023-08-15 21:38:02 +02:00
ui . req . ContentLength = & size
2022-05-03 18:39:01 +02:00
}
2022-01-28 17:18:40 +01:00
if md5sumBase64 != "" {
2023-08-15 21:38:02 +02:00
ui . req . ContentMD5 = & md5sumBase64
2019-12-31 00:17:06 +01:00
}
2020-12-03 03:30:06 +01:00
if o . fs . opt . RequesterPays {
2024-08-03 12:35:32 +02:00
ui . req . RequestPayer = types . RequestPayerRequester
2020-12-03 03:30:06 +01:00
}
2019-12-31 00:17:06 +01:00
if o . fs . opt . ServerSideEncryption != "" {
2024-08-03 12:35:32 +02:00
ui . req . ServerSideEncryption = types . ServerSideEncryption ( o . fs . opt . ServerSideEncryption )
2019-12-31 00:17:06 +01:00
}
2020-03-30 12:26:52 +02:00
if o . fs . opt . SSECustomerAlgorithm != "" {
2023-08-15 21:38:02 +02:00
ui . req . SSECustomerAlgorithm = & o . fs . opt . SSECustomerAlgorithm
2020-03-30 12:26:52 +02:00
}
2024-08-07 11:14:58 +02:00
if o . fs . opt . SSECustomerKeyBase64 != "" {
ui . req . SSECustomerKey = & o . fs . opt . SSECustomerKeyBase64
2020-03-30 12:26:52 +02:00
}
if o . fs . opt . SSECustomerKeyMD5 != "" {
2023-08-15 21:38:02 +02:00
ui . req . SSECustomerKeyMD5 = & o . fs . opt . SSECustomerKeyMD5
2020-03-30 12:26:52 +02:00
}
2019-12-31 00:17:06 +01:00
if o . fs . opt . SSEKMSKeyID != "" {
2023-08-15 21:38:02 +02:00
ui . req . SSEKMSKeyId = & o . fs . opt . SSEKMSKeyID
2019-12-31 00:17:06 +01:00
}
if o . fs . opt . StorageClass != "" {
2024-08-03 12:35:32 +02:00
ui . req . StorageClass = types . StorageClass ( o . fs . opt . StorageClass )
2019-12-31 00:17:06 +01:00
}
2020-06-05 12:45:54 +02:00
// Apply upload options
for _ , option := range options {
key , value := option . Header ( )
lowerKey := strings . ToLower ( key )
switch lowerKey {
case "" :
// ignore
case "cache-control" :
2023-08-15 21:38:02 +02:00
ui . req . CacheControl = aws . String ( value )
2020-06-05 12:45:54 +02:00
case "content-disposition" :
2023-08-15 21:38:02 +02:00
ui . req . ContentDisposition = aws . String ( value )
2020-06-05 12:45:54 +02:00
case "content-encoding" :
2023-08-15 21:38:02 +02:00
ui . req . ContentEncoding = aws . String ( value )
2020-06-05 12:45:54 +02:00
case "content-language" :
2023-08-15 21:38:02 +02:00
ui . req . ContentLanguage = aws . String ( value )
2020-06-05 12:45:54 +02:00
case "content-type" :
2023-08-15 21:38:02 +02:00
ui . req . ContentType = aws . String ( value )
2020-06-05 12:45:54 +02:00
case "x-amz-tagging" :
2023-08-15 21:38:02 +02:00
ui . req . Tagging = aws . String ( value )
2020-06-05 12:45:54 +02:00
default :
const amzMetaPrefix = "x-amz-meta-"
if strings . HasPrefix ( lowerKey , amzMetaPrefix ) {
metaKey := lowerKey [ len ( amzMetaPrefix ) : ]
2024-08-03 12:35:32 +02:00
ui . req . Metadata [ metaKey ] = value
2020-06-05 12:45:54 +02:00
} else {
fs . Errorf ( o , "Don't know how to set key %q on upload" , key )
}
}
}
2019-12-31 00:17:06 +01:00
2022-10-11 18:54:22 +02:00
// Check metadata keys and values are valid
2023-08-15 21:38:02 +02:00
for key , value := range ui . req . Metadata {
2022-10-11 18:54:22 +02:00
if ! httpguts . ValidHeaderFieldName ( key ) {
fs . Errorf ( o , "Dropping invalid metadata key %q" , key )
2023-08-15 21:38:02 +02:00
delete ( ui . req . Metadata , key )
2024-08-03 12:35:32 +02:00
} else if ! httpguts . ValidHeaderFieldValue ( value ) {
fs . Errorf ( o , "Dropping invalid metadata value %q for key %q" , value , key )
2023-08-15 21:38:02 +02:00
delete ( ui . req . Metadata , key )
2022-10-11 18:54:22 +02:00
}
}
2023-08-15 21:38:02 +02:00
return ui , nil
2023-07-18 20:37:31 +02:00
}
// Update the Object from in with modTime and size
func ( o * Object ) Update ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) error {
if o . fs . opt . VersionAt . IsSet ( ) {
return errNotWithVersionAt
}
size := src . Size ( )
multipart := size < 0 || size >= int64 ( o . fs . opt . UploadCutoff )
2022-05-03 18:39:01 +02:00
var wantETag string // Multipart upload Etag to check
2023-02-13 11:31:31 +01:00
var gotETag string // Etag we got from the upload
2022-05-03 18:39:01 +02:00
var lastModified time . Time // Time we got from the upload
2022-07-25 17:06:15 +02:00
var versionID * string // versionID we got from the upload
2023-07-18 20:37:31 +02:00
var err error
2023-08-15 21:38:02 +02:00
var ui uploadInfo
2018-11-26 22:09:23 +01:00
if multipart {
2023-10-06 17:53:36 +02:00
wantETag , gotETag , versionID , ui , err = o . uploadMultipart ( ctx , src , in , options ... )
2018-11-26 22:09:23 +01:00
} else {
2024-03-05 18:21:06 +01:00
ui , err = o . prepareUpload ( ctx , src , options , false )
2023-07-18 20:37:31 +02:00
if err != nil {
2023-08-15 21:38:02 +02:00
return fmt . Errorf ( "failed to prepare upload: %w" , err )
2023-07-18 20:37:31 +02:00
}
2022-05-03 18:39:01 +02:00
if o . fs . opt . UsePresignedRequest {
2023-08-15 21:38:02 +02:00
gotETag , lastModified , versionID , err = o . uploadSinglepartPresignedRequest ( ctx , ui . req , size , in )
2022-05-03 18:39:01 +02:00
} else {
2023-08-15 21:38:02 +02:00
gotETag , lastModified , versionID , err = o . uploadSinglepartPutObject ( ctx , ui . req , size , in )
2018-11-26 22:09:23 +01:00
}
2014-07-19 13:37:11 +02:00
}
2022-05-03 18:39:01 +02:00
if err != nil {
return err
}
2022-12-16 12:44:58 +01:00
// Only record versionID if we are using --s3-versions or --s3-version-at
if o . fs . opt . Versions || o . fs . opt . VersionAt . IsSet ( ) {
o . versionID = versionID
} else {
o . versionID = nil
}
2014-12-23 13:09:02 +01:00
2021-01-29 12:13:42 +01:00
// User requested we don't HEAD the object after uploading it
// so make up the object as best we can assuming it got
// uploaded properly. If size < 0 then we need to do the HEAD.
2023-02-13 11:31:31 +01:00
var head * s3 . HeadObjectOutput
2021-01-29 12:13:42 +01:00
if o . fs . opt . NoHead && size >= 0 {
2023-02-13 11:31:31 +01:00
head = new ( s3 . HeadObjectOutput )
//structs.SetFrom(head, &req)
2023-08-15 21:38:02 +02:00
setFrom_s3HeadObjectOutput_s3PutObjectInput ( head , ui . req )
head . ETag = & ui . md5sumHex // doesn't matter quotes are missing
2022-05-24 13:32:39 +02:00
head . ContentLength = & size
2023-02-13 11:31:31 +01:00
// We get etag back from single and multipart upload so fill it in here
if gotETag != "" {
head . ETag = & gotETag
2022-05-03 18:39:01 +02:00
}
2022-05-24 13:32:39 +02:00
if lastModified . IsZero ( ) {
lastModified = time . Now ( )
2021-01-29 12:13:42 +01:00
}
2022-05-24 13:32:39 +02:00
head . LastModified = & lastModified
2022-07-25 17:06:15 +02:00
head . VersionId = versionID
2023-02-13 11:31:31 +01:00
} else {
// Read the metadata from the newly created object
o . meta = nil // wipe old metadata
head , err = o . headObject ( ctx )
if err != nil {
return err
}
2022-02-16 18:50:11 +01:00
}
2022-05-24 13:32:39 +02:00
o . setMetaData ( head )
2023-02-13 11:31:31 +01:00
// Check multipart upload ETag if required
2022-02-27 16:47:31 +01:00
if o . fs . opt . UseMultipartEtag . Value && ! o . fs . etagIsNotMD5 && wantETag != "" && head . ETag != nil && * head . ETag != "" {
2022-02-16 18:50:11 +01:00
gotETag := strings . Trim ( strings . ToLower ( * head . ETag ) , ` " ` )
if wantETag != gotETag {
return fmt . Errorf ( "multipart upload corrupted: Etag differ: expecting %s but got %s" , wantETag , gotETag )
}
fs . Debugf ( o , "Multipart upload Etag: %s OK" , wantETag )
}
2014-04-18 18:04:21 +02:00
return err
}
2013-01-08 19:53:35 +01:00
// Remove an object
2019-06-17 10:34:30 +02:00
func ( o * Object ) Remove ( ctx context . Context ) error {
2022-07-26 18:58:57 +02:00
if o . fs . opt . VersionAt . IsSet ( ) {
return errNotWithVersionAt
}
2019-08-09 12:29:36 +02:00
bucket , bucketPath := o . split ( )
2014-12-23 13:09:02 +01:00
req := s3 . DeleteObjectInput {
2022-07-25 17:06:15 +02:00
Bucket : & bucket ,
Key : & bucketPath ,
VersionId : o . versionID ,
2014-12-23 13:09:02 +01:00
}
2020-12-03 03:30:06 +01:00
if o . fs . opt . RequesterPays {
2024-08-03 12:35:32 +02:00
req . RequestPayer = types . RequestPayerRequester
2020-12-03 03:30:06 +01:00
}
2018-09-03 06:41:04 +02:00
err := o . fs . pacer . Call ( func ( ) ( bool , error ) {
2024-08-03 12:35:32 +02:00
_ , err := o . fs . c . DeleteObject ( ctx , & req )
2021-03-16 16:50:02 +01:00
return o . fs . shouldRetry ( ctx , err )
2018-09-03 06:41:04 +02:00
} )
2014-12-23 13:09:02 +01:00
return err
2013-01-08 19:53:35 +01:00
}
2016-09-21 23:13:24 +02:00
// MimeType of an Object if known, "" otherwise
2019-06-17 10:34:30 +02:00
func ( o * Object ) MimeType ( ctx context . Context ) string {
err := o . readMetaData ( ctx )
2016-09-21 23:13:24 +02:00
if err != nil {
2017-02-09 12:01:20 +01:00
fs . Logf ( o , "Failed to read metadata: %v" , err )
2016-09-21 23:13:24 +02:00
return ""
}
return o . mimeType
}
2019-09-09 21:44:50 +02:00
// SetTier performs changing storage class
func ( o * Object ) SetTier ( tier string ) ( err error ) {
ctx := context . TODO ( )
tier = strings . ToUpper ( tier )
bucket , bucketPath := o . split ( )
req := s3 . CopyObjectInput {
2024-08-03 12:35:32 +02:00
MetadataDirective : types . MetadataDirectiveCopy ,
StorageClass : types . StorageClass ( tier ) ,
2019-09-09 21:44:50 +02:00
}
2020-07-30 11:52:32 +02:00
err = o . fs . copy ( ctx , & req , bucket , bucketPath , bucket , bucketPath , o )
2019-09-09 21:44:50 +02:00
if err != nil {
return err
}
2022-05-24 13:32:39 +02:00
o . storageClass = & tier
2019-09-09 21:44:50 +02:00
return err
}
// GetTier returns storage class as string
func ( o * Object ) GetTier ( ) string {
2022-05-24 13:32:39 +02:00
if o . storageClass == nil || * o . storageClass == "" {
2019-09-09 21:44:50 +02:00
return "STANDARD"
}
2022-05-24 13:32:39 +02:00
return * o . storageClass
}
// Metadata returns metadata for an object
//
// It should return nil if there is no Metadata
func ( o * Object ) Metadata ( ctx context . Context ) ( metadata fs . Metadata , err error ) {
err = o . readMetaData ( ctx )
if err != nil {
return nil , err
}
metadata = make ( fs . Metadata , len ( o . meta ) + 7 )
for k , v := range o . meta {
switch k {
case metaMtime :
if modTime , err := swift . FloatStringToTime ( v ) ; err == nil {
metadata [ "mtime" ] = modTime . Format ( time . RFC3339Nano )
}
case metaMD5Hash :
// don't write hash metadata
default :
metadata [ k ] = v
}
}
if o . mimeType != "" {
metadata [ "content-type" ] = o . mimeType
}
// metadata["x-amz-tagging"] = ""
if ! o . lastModified . IsZero ( ) {
metadata [ "btime" ] = o . lastModified . Format ( time . RFC3339Nano )
}
// Set system metadata
setMetadata := func ( k string , v * string ) {
2022-10-12 09:55:58 +02:00
if o . fs . opt . NoSystemMetadata {
return
}
2022-05-24 13:32:39 +02:00
if v == nil || * v == "" {
return
}
metadata [ k ] = * v
}
setMetadata ( "cache-control" , o . cacheControl )
setMetadata ( "content-disposition" , o . contentDisposition )
setMetadata ( "content-encoding" , o . contentEncoding )
setMetadata ( "content-language" , o . contentLanguage )
2023-04-27 16:27:31 +02:00
metadata [ "tier" ] = o . GetTier ( )
2022-05-24 13:32:39 +02:00
return metadata , nil
2019-09-09 21:44:50 +02:00
}
2013-01-08 19:53:35 +01:00
// Check the interfaces are satisfied
2015-11-07 12:14:46 +01:00
var (
2023-08-15 21:38:02 +02:00
_ fs . Fs = & Fs { }
_ fs . Purger = & Fs { }
_ fs . Copier = & Fs { }
_ fs . PutStreamer = & Fs { }
_ fs . ListRer = & Fs { }
_ fs . Commander = & Fs { }
_ fs . CleanUpper = & Fs { }
_ fs . OpenChunkWriter = & Fs { }
_ fs . Object = & Object { }
_ fs . MimeTyper = & Object { }
_ fs . GetTierer = & Object { }
_ fs . SetTierer = & Object { }
_ fs . Metadataer = & Object { }
2015-11-07 12:14:46 +01:00
)