extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-08-16 08:38:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into 7935-sftp-read-ssh-config-file

Browse Source
This commit is contained in:
kivi
2024-10-24 12:15:36 +02:00
committed by GitHub
parent 4cc9b25bdf 175aa07cdd
commit 1ef92ef7f4
11 changed files with 269 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -111,6 +111,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
* Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
* Seafile [:page_facing_up:](https://rclone.org/seafile/)
* SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
* Selectel Object Storage [:page_facing_up:](https://rclone.org/s3/#selectel)
* SFTP [:page_facing_up:](https://rclone.org/sftp/)
* SMB / CIFS [:page_facing_up:](https://rclone.org/smb/)
* StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)

103
backend/drive/drive.go
View File

@ -3559,7 +3559,8 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
return nil
}
func (f *Fs) query(ctx context.Context, query string) (entries []*drive.File, err error) {
// Run the drive query calling fn on each entry found
func (f *Fs) queryFn(ctx context.Context, query string, fn func(*drive.File)) (err error) {
list := f.svc.Files.List()
if query != "" {
list.Q(query)
@ -3578,10 +3579,7 @@ func (f *Fs) query(ctx context.Context, query string) (entries []*drive.File, er
if f.rootFolderID == "appDataFolder" {
list.Spaces("appDataFolder")
}
fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.getFileFields(ctx))
var results []*drive.File
for {
var files *drive.FileList
err = f.pacer.Call(func() (bool, error) {
@ -3589,20 +3587,66 @@ func (f *Fs) query(ctx context.Context, query string) (entries []*drive.File, er
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, fmt.Errorf("failed to execute query: %w", err)
return fmt.Errorf("failed to execute query: %w", err)
}
if files.IncompleteSearch {
fs.Errorf(f, "search result INCOMPLETE")
}
results = append(results, files.Files...)
for _, item := range files.Files {
fn(item)
}
if files.NextPageToken == "" {
break
}
list.PageToken(files.NextPageToken)
}
return nil
}
// Run the drive query returning the entries found
func (f *Fs) query(ctx context.Context, query string) (entries []*drive.File, err error) {
var results []*drive.File
err = f.queryFn(ctx, query, func(item *drive.File) {
results = append(results, item)
})
if err != nil {
return nil, err
}
return results, nil
}
// Rescue, list or delete orphaned files
func (f *Fs) rescue(ctx context.Context, dirID string, delete bool) (err error) {
return f.queryFn(ctx, "'me' in owners and trashed=false", func(item *drive.File) {
if len(item.Parents) != 0 {
return
}
// Have found an orphaned entry
if delete {
fs.Infof(item.Name, "Deleting orphan %q into trash", item.Id)
err = f.delete(ctx, item.Id, true)
if err != nil {
fs.Errorf(item.Name, "Failed to delete orphan %q: %v", item.Id, err)
}
} else if dirID == "" {
operations.SyncPrintf("%q, %q\n", item.Name, item.Id)
} else {
fs.Infof(item.Name, "Rescuing orphan %q", item.Id)
err = f.pacer.Call(func() (bool, error) {
_, err = f.svc.Files.Update(item.Id, nil).
AddParents(dirID).
Fields(f.getFileFields(ctx)).
SupportsAllDrives(true).
Context(ctx).Do()
return f.shouldRetry(ctx, err)
})
if err != nil {
fs.Errorf(item.Name, "Failed to rescue orphan %q: %v", item.Id, err)
}
}
})
}
var commandHelp = []fs.CommandHelp{{
Name: "get",
Short: "Get command for fetching the drive config parameters",
@ -3794,6 +3838,37 @@ The result is a JSON array of matches, for example:
"webViewLink": "https://drive.google.com/file/d/0AxBe_CDEF4zkGHI4d0FjYko2QkD/view?usp=drivesdk\u0026resourcekey=0-ABCDEFGHIXJQpIGqBJq3MC"
}
]`,
}, {
Name: "rescue",
Short: "Rescue or delete any orphaned files",
Long: `This command rescues or deletes any orphaned files or directories.
Sometimes files can get orphaned in Google Drive. This means that they
are no longer in any folder in Google Drive.
This command finds those files and either rescues them to a directory
you specify or deletes them.
Usage:
This can be used in 3 ways.
First, list all orphaned files
rclone backend rescue drive:
Second rescue all orphaned files to the directory indicated
rclone backend rescue drive: "relative/path/to/rescue/directory"
e.g. To rescue all orphans to a directory called "Orphans" in the top level
rclone backend rescue drive: Orphans
Third delete all orphaned files to the trash
rclone backend rescue drive: -o delete
`,
}}
// Command the backend to run a named command
@ -3922,6 +3997,22 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
} else {
return nil, errors.New("need a query argument")
}
case "rescue":
dirID := ""
_, delete := opt["delete"]
if len(arg) == 0 {
// no arguments - list only
} else if !delete && len(arg) == 1 {
dir := arg[0]
dirID, err = f.dirCache.FindDir(ctx, dir, true)
if err != nil {
return nil, fmt.Errorf("failed to find or create rescue directory %q: %w", dir, err)
}
fs.Infof(f, "Rescuing orphans into %q", dir)
} else {
return nil, errors.New("syntax error: need 0 or 1 args or -o delete")
}
return nil, f.rescue(ctx, dirID, delete)
default:
return nil, fs.ErrorCommandNotFound
}

2
backend/onedrive/onedrive.go
View File

@ -827,7 +827,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
retry = true
fs.Debugf(nil, "HTTP 401: Unable to initialize RPS. Trying again.")
}
case 429: // Too Many Requests.
case 429, 503: // Too Many Requests, Server Too Busy
// see https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
if values := resp.Header["Retry-After"]; len(values) == 1 && values[0] != "" {
retryAfter, parseErr := strconv.Atoi(values[0])

39
backend/s3/s3.go
View File

@ -154,6 +154,9 @@ var providerOption = fs.Option{
}, {
Value: "SeaweedFS",
Help: "SeaweedFS S3",
}, {
Value: "Selectel",
Help: "Selectel Object Storage",
}, {
Value: "StackPath",
Help: "StackPath Object Storage",
@ -551,10 +554,19 @@ func init() {
Value: "tw-001",
Help: "Asia (Taiwan)",
}},
}, {
// See endpoints for object storage regions: https://docs.selectel.ru/en/cloud/object-storage/manage/domains/#s3-api-domains
Name: "region",
Help: "Region where your data stored.\n",
Provider: "Selectel",
Examples: []fs.OptionExample{{
Value: "ru-1",
Help: "St. Petersburg",
}},
}, {
Name: "region",
Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Storj,Synology,TencentCOS,HuaweiOBS,IDrive",
Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive",
Examples: []fs.OptionExample{{
Value: "",
Help: "Use this if unsure.\nWill use v4 signatures and an empty region.",
@ -1319,10 +1331,19 @@ func init() {
Value: "s3-ap-northeast-1.qiniucs.com",
Help: "Northeast Asia Endpoint 1",
}},
}, {
// Selectel endpoints: https://docs.selectel.ru/en/cloud/object-storage/manage/domains/#s3-api-domains
Name: "endpoint",
Help: "Endpoint for Selectel Object Storage.",
Provider: "Selectel",
Examples: []fs.OptionExample{{
Value: "s3.ru-1.storage.selcloud.ru",
Help: "Saint Petersburg",
}},
}, {
Name: "endpoint",
Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,MagaluCloud,Scaleway,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,MagaluCloud,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
Examples: []fs.OptionExample{{
Value: "objects-us-east-1.dream.io",
Help: "Dream Objects endpoint",
@ -1423,6 +1444,10 @@ func init() {
Value: "s3.eu-west-2.wasabisys.com",
Help: "Wasabi EU West 2 (Paris)",
Provider: "Wasabi",
}, {
Value: "s3.eu-south-1.wasabisys.com",
Help: "Wasabi EU South 1 (Milan)",
Provider: "Wasabi",
}, {
Value: "s3.ap-northeast-1.wasabisys.com",
Help: "Wasabi AP Northeast 1 (Tokyo) endpoint",
@ -1841,7 +1866,7 @@ func init() {
}, {
Name: "location_constraint",
Help: "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
Provider: "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Leviia,Liara,Linode,Magalu,Outscale,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS,Petabox",
Provider: "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Leviia,Liara,Linode,Magalu,Outscale,Qiniu,RackCorp,Scaleway,Selectel,StackPath,Storj,TencentCOS,Petabox",
}, {
Name: "acl",
Help: `Canned ACL used when creating buckets and storing or copying objects.
@ -1856,7 +1881,7 @@ doesn't copy the ACL from the source but rather writes a fresh one.
If the acl is an empty string then no X-Amz-Acl: header is added and
the default (private) will be used.
`,
Provider: "!Storj,Synology,Cloudflare",
Provider: "!Storj,Selectel,Synology,Cloudflare",
Examples: []fs.OptionExample{{
Value: "default",
Help: "Owner gets Full_CONTROL.\nNo one else has access rights (default).",
@ -3426,6 +3451,8 @@ func setQuirks(opt *Options) {
}
urlEncodeListings = true
useAlreadyExists = true
case "Selectel":
urlEncodeListings = false
case "SeaweedFS":
listObjectsV2 = false // untested
virtualHostStyle = false
@ -3443,6 +3470,10 @@ func setQuirks(opt *Options) {
opt.ChunkSize = 64 * fs.Mebi
}
useAlreadyExists = false // returns BucketAlreadyExists
// Storj doesn't support multi-part server side copy:
// https://github.com/storj/roadmap/issues/40
// So make cutoff very large which it does support
opt.CopyCutoff = math.MaxInt64
case "Synology":
useMultipartEtag = false
useAlreadyExists = false // untested

2
cmd/serve/dlna/dlna_util.go
View File

@ -108,7 +108,7 @@ func (lrw *loggingResponseWriter) logRequest(code int, err interface{}) {
err = ""
}
fs.LogPrintf(level, lrw.request.URL, "%s %s %d %s %s",
fs.LogLevelPrintf(level, lrw.request.URL, "%s %s %d %s %s",
lrw.request.RemoteAddr, lrw.request.Method, code,
lrw.request.Header.Get("SOAPACTION"), err)
}

1
docs/content/_index.md
View File

@ -178,6 +178,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="Seafile" home="https://www.seafile.com/" config="/seafile/" >}}
{{< provider name="Seagate Lyve Cloud" home="https://www.seagate.com/gb/en/services/cloud/storage/" config="/s3/#lyve" >}}
{{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}}
{{< provider name="Selectel" home="https://selectel.ru/services/cloud/storage/" config="/s3/#selectel" >}}
{{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol" config="/sftp/" >}}
{{< provider name="Sia" home="https://sia.tech/" config="/sia/" >}}
{{< provider name="SMB / CIFS" home="https://en.wikipedia.org/wiki/Server_Message_Block" config="/smb/" >}}

5
docs/content/authors.md
View File

@ -900,3 +900,8 @@ put them back in again.` >}}
* lostb1t <coding-mosses0z@icloud.com>
* Matthias Gatto <matthias.gatto@outscale.com>
* André Tran <andre.tran@outscale.com>
* Simon Bos <simon@simonbos.be>
* Alexandre Hamez <199517+ahamez@users.noreply.github.com>
* Randy Bush <randy@psg.com>
* Diego Monti <diegmonti@users.noreply.github.com>
* tgfisher <tgfisher@stanford.edu>

4
docs/content/drive.md
View File

@ -1810,9 +1810,9 @@ then select "OAuth client ID".
9. It will show you a client ID and client secret. Make a note of these.
(If you selected "External" at Step 5 continue to Step 9.
(If you selected "External" at Step 5 continue to Step 10.
If you chose "Internal" you don't need to publish and can skip straight to
Step 10 but your destination drive must be part of the same Google Workspace.)
Step 11 but your destination drive must be part of the same Google Workspace.)
10. Go to "Oauth consent screen" and then click "PUBLISH APP" button and confirm.
You will also want to add yourself as a test user.

3
docs/content/filtering.md
View File

@ -505,6 +505,8 @@ processed in.
Arrange the order of filter rules with the most restrictive first and
work down.
Lines starting with # or ; are ignored, and can be used to write comments. Inline comments are not supported. _Use `-vv --dump filters` to see how they appear in the final regexp._
E.g. for `filter-file.txt`:
# a sample filter rule file
@ -512,6 +514,7 @@ E.g. for `filter-file.txt`:
+ *.jpg
+ *.png
+ file2.avi
- /dir/tmp/** # WARNING! This text will be treated as part of the path.
- /dir/Trash/**
+ /dir/**
# exclude everything else

124
docs/content/s3.md
View File

@ -35,6 +35,7 @@ The S3 backend can be used with a number of different providers:
{{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}}
{{< provider name="Seagate Lyve Cloud" home="https://www.seagate.com/gb/en/services/cloud/storage/" config="/s3/#lyve" >}}
{{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}}
{{< provider name="Selectel" home="https://selectel.ru/services/cloud/storage/" config="/s3/#selectel" >}}
{{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
{{< provider name="Storj" home="https://storj.io/" config="/s3/#storj" >}}
{{< provider name="Synology C2 Object Storage" home="https://c2.synology.com/en-global/object-storage/overview" config="/s3/#synology-c2" >}}
@ -3654,8 +3655,8 @@ chunk_size = 5M
copy_cutoff = 5M
```
[C14 Cold Storage](https://www.online.net/en/storage/c14-cold-storage) is the low-cost S3 Glacier alternative from Scaleway and it works the same way as on S3 by accepting the "GLACIER" `storage_class`.
So you can configure your remote with the `storage_class = GLACIER` option to upload directly to C14. Don't forget that in this state you can't read files back after, you will need to restore them to "STANDARD" storage_class first before being able to read them (see "restore" section above)
[Scaleway Glacier](https://www.scaleway.com/en/glacier-cold-storage/) is the low-cost S3 Glacier alternative from Scaleway and it works the same way as on S3 by accepting the "GLACIER" `storage_class`.
So you can configure your remote with the `storage_class = GLACIER` option to upload directly to Scaleway Glacier. Don't forget that in this state you can't read files back after, you will need to restore them to "STANDARD" storage_class first before being able to read them (see "restore" section above)
### Seagate Lyve Cloud {#lyve}
@ -3850,6 +3851,125 @@ So once set up, for example to copy files into a bucket
rclone copy /path/to/files seaweedfs_s3:foo
```
### Selectel
[Selectel Cloud Storage](https://selectel.ru/services/cloud/storage/)
is an S3 compatible storage system which features triple redundancy
storage, automatic scaling, high availability and a comprehensive IAM
system.
Selectel have a section on their website for [configuring
rclone](https://docs.selectel.ru/en/cloud/object-storage/tools/rclone/)
which shows how to make the right API keys.
From rclone v1.69 Selectel is a supported operator - please choose the
`Selectel` provider type.
Note that you should use "vHosted" access for the buckets (which is
the recommended default), not "path style".
You can use `rclone config` to make a new provider like this
```
No remotes found, make a new one?
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
Enter name for new remote.
name> selectel
Option Storage.
Type of storage to configure.
Choose a number from below, or type in your own value.
[snip]
XX / Amazon S3 Compliant Storage Providers including ..., Selectel, ...
\ (s3)
[snip]
Storage> s3
Option provider.
Choose your S3 provider.
Choose a number from below, or type in your own value.
Press Enter to leave empty.
[snip]
XX / Selectel Object Storage
\ (Selectel)
[snip]
provider> Selectel
Option env_auth.
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
Only applies if access_key_id and secret_access_key is blank.
Choose a number from below, or type in your own boolean value (true or false).
Press Enter for the default (false).
1 / Enter AWS credentials in the next step.
\ (false)
2 / Get AWS credentials from the environment (env vars or IAM).
\ (true)
env_auth> 1
Option access_key_id.
AWS Access Key ID.
Leave blank for anonymous access or runtime credentials.
Enter a value. Press Enter to leave empty.
access_key_id> ACCESS_KEY
Option secret_access_key.
AWS Secret Access Key (password).
Leave blank for anonymous access or runtime credentials.
Enter a value. Press Enter to leave empty.
secret_access_key> SECRET_ACCESS_KEY
Option region.
Region where your data stored.
Choose a number from below, or type in your own value.
Press Enter to leave empty.
1 / St. Petersburg
\ (ru-1)
region> 1
Option endpoint.
Endpoint for Selectel Object Storage.
Choose a number from below, or type in your own value.
Press Enter to leave empty.
1 / Saint Petersburg
\ (s3.ru-1.storage.selcloud.ru)
endpoint> 1
Edit advanced config?
y) Yes
n) No (default)
y/n> n
Configuration complete.
Options:
- type: s3
- provider: Selectel
- access_key_id: ACCESS_KEY
- secret_access_key: SECRET_ACCESS_KEY
- region: ru-1
- endpoint: s3.ru-1.storage.selcloud.ru
Keep this "selectel" remote?
y) Yes this is OK (default)
e) Edit this remote
d) Delete this remote
y/e/d> y
```
And your config should end up looking like this:
```
[selectel]
type = s3
provider = Selectel
access_key_id = ACCESS_KEY
secret_access_key = SECRET_ACCESS_KEY
region = ru-1
endpoint = s3.ru-1.storage.selcloud.ru
```
### Wasabi
[Wasabi](https://wasabi.com) is a cloud-based object storage service for a

2
fs/open_options.go
View File

@ -293,7 +293,7 @@ type ChunkOption struct {
// Header formats the option as an http header
func (o *ChunkOption) Header() (key string, value string) {
return "chunkSize", fmt.Sprintf("%v", o.ChunkSize)
return "", ""
}
// Mandatory returns whether the option must be parsed or can be ignored