drive: added moveid function to backend

This commit is contained in:
Spencer McCullough 2024-11-20 13:35:16 -07:00
parent d8bc542ffc
commit 6b1bf5fbc0
3 changed files with 566 additions and 439 deletions

View File

@ -3559,6 +3559,41 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
return nil
}
// move file with id to dest (original file is deleted, unlike with copy)
func (f *Fs) moveID(ctx context.Context, id, dest string) (err error) {
info, err := f.getFile(ctx, id, f.getFileFields(ctx))
if err != nil {
return fmt.Errorf("couldn't find id: %w", err)
}
if info.MimeType == driveFolderType {
return fmt.Errorf("can't copy directory use: rclone copy --drive-root-folder-id %s %s %s", id, fs.ConfigString(f), dest)
}
info.Name = f.opt.Enc.ToStandardName(info.Name)
o, err := f.newObjectWithInfo(ctx, info.Name, info)
if err != nil {
return err
}
destDir, destLeaf, err := fspath.Split(dest)
if err != nil {
return err
}
if destLeaf == "" {
destLeaf = path.Base(o.Remote())
}
if destDir == "" {
destDir = "."
}
dstFs, err := cache.Get(ctx, destDir)
if err != nil {
return err
}
_, err = operations.Move(ctx, dstFs, nil, destLeaf, o)
if err != nil {
return fmt.Errorf("move failed: %w", err)
}
return nil
}
// Run the drive query calling fn on each entry found
func (f *Fs) queryFn(ctx context.Context, query string, fn func(*drive.File)) (err error) {
list := f.svc.Files.List()
@ -3789,6 +3824,29 @@ component will be used as the file name.
If the destination is a drive backend then server-side copying will be
attempted if possible.
Use the --interactive/-i or --dry-run flag to see what would be copied before copying.
`,
}, {
Name: "moveid",
Short: "Move files by ID",
Long: `This command copies files by ID
Usage:
rclone backend moveid drive: ID path
rclone backend moveid drive: ID1 path1 ID2 path2
It moves the drive file with ID given to the path (an rclone path which
will be passed internally to rclone copyto). The ID and path pairs can be
repeated.
The path should end with a / to indicate copy the file as named to
this directory. If it doesn't end with a / then the last path
component will be used as the file name.
If the destination is a drive backend then server-side copying will be
attempted if possible.
Use the --interactive/-i or --dry-run flag to see what would be copied before copying.
`,
}, {
@ -3982,6 +4040,19 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
}
}
return nil, nil
case "moveid":
if len(arg)%2 != 0 {
return nil, errors.New("need an even number of arguments")
}
for len(arg) > 0 {
id, dest := arg[0], arg[1]
arg = arg[2:]
err = f.moveID(ctx, id, dest)
if err != nil {
return nil, fmt.Errorf("failed moving %q to %q: %w", id, dest, err)
}
}
return nil, nil
case "exportformats":
return f.exportFormats(ctx), nil
case "importformats":

View File

@ -524,6 +524,51 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
})
}
// TestIntegration/FsMkdir/FsPutFiles/Internal/MoveID
func (f *Fs) InternalTestMoveID(t *testing.T) {
ctx := context.Background()
obj, err := f.NewObject(ctx, existingFile)
require.NoError(t, err)
o := obj.(*Object)
dir := t.TempDir()
checkFile := func(name string) {
filePath := filepath.Join(dir, name)
fi, err := os.Stat(filePath)
require.NoError(t, err)
assert.Equal(t, int64(100), fi.Size())
err = os.Remove(filePath)
require.NoError(t, err)
}
t.Run("BadID", func(t *testing.T) {
err = f.moveID(ctx, "ID-NOT-FOUND", dir+"/")
require.Error(t, err)
assert.Contains(t, err.Error(), "couldn't find id")
})
t.Run("Directory", func(t *testing.T) {
rootID, err := f.dirCache.RootID(ctx, false)
require.NoError(t, err)
err = f.moveID(ctx, rootID, dir+"/")
require.Error(t, err)
assert.Contains(t, err.Error(), "can't copy directory")
})
t.Run("WithoutDestName", func(t *testing.T) {
err = f.moveID(ctx, o.id, dir+"/")
require.NoError(t, err)
checkFile(path.Base(existingFile))
})
t.Run("WithDestName", func(t *testing.T) {
err = f.moveID(ctx, o.id, dir+"/potato.txt")
require.NoError(t, err)
checkFile("potato.txt")
})
}
// TestIntegration/FsMkdir/FsPutFiles/Internal/Query
func (f *Fs) InternalTestQuery(t *testing.T) {
ctx := context.Background()
@ -648,6 +693,7 @@ func (f *Fs) InternalTest(t *testing.T) {
t.Run("Shortcuts", f.InternalTestShortcuts)
t.Run("UnTrash", f.InternalTestUnTrash)
t.Run("CopyID", f.InternalTestCopyID)
t.Run("MoveID", f.InternalTestMoveID)
t.Run("Query", f.InternalTestQuery)
t.Run("AgeQuery", f.InternalTestAgeQuery)
t.Run("ShouldRetry", f.InternalTestShouldRetry)

View File

@ -225,38 +225,38 @@ There's a few steps we need to go through to accomplish this:
##### 1. Create a service account for example.com
- To create a service account and obtain its credentials, go to the
[Google Developer Console](https://console.developers.google.com).
- You must have a project - create one if you don't and make sure you are on the selected project.
- Then go to "IAM & admin" -> "Service Accounts".
- Use the "Create Service Account" button. Fill in "Service account name"
and "Service account ID" with something that identifies your client.
- Select "Create And Continue". Step 2 and 3 are optional.
- Click on the newly created service account
- Click "Keys" and then "Add Key" and then "Create new key"
- Choose type "JSON" and click create
- This will download a small JSON file that rclone will use for authentication.
- To create a service account and obtain its credentials, go to the
[Google Developer Console](https://console.developers.google.com).
- You must have a project - create one if you don't and make sure you are on the selected project.
- Then go to "IAM & admin" -> "Service Accounts".
- Use the "Create Service Account" button. Fill in "Service account name"
and "Service account ID" with something that identifies your client.
- Select "Create And Continue". Step 2 and 3 are optional.
- Click on the newly created service account
- Click "Keys" and then "Add Key" and then "Create new key"
- Choose type "JSON" and click create
- This will download a small JSON file that rclone will use for authentication.
If you ever need to remove access, press the "Delete service
account key" button.
##### 2. Allowing API access to example.com Google Drive
- Go to example.com's [Workspace Admin Console](https://admin.google.com)
- Go into "Security" (or use the search bar)
- Select "Access and data control" and then "API controls"
- Click "Manage domain-wide delegation"
- Click "Add new"
- In the "Client ID" field enter the service account's
"Client ID" - this can be found in the Developer Console under
"IAM & Admin" -> "Service Accounts", then "View Client ID" for
the newly created service account.
It is a ~21 character numerical string.
- In the next field, "OAuth Scopes", enter
`https://www.googleapis.com/auth/drive`
to grant read/write access to Google Drive specifically.
You can also use `https://www.googleapis.com/auth/drive.readonly` for read only access.
- Click "Authorise"
- Go to example.com's [Workspace Admin Console](https://admin.google.com)
- Go into "Security" (or use the search bar)
- Select "Access and data control" and then "API controls"
- Click "Manage domain-wide delegation"
- Click "Add new"
- In the "Client ID" field enter the service account's
"Client ID" - this can be found in the Developer Console under
"IAM & Admin" -> "Service Accounts", then "View Client ID" for
the newly created service account.
It is a ~21 character numerical string.
- In the next field, "OAuth Scopes", enter
`https://www.googleapis.com/auth/drive`
to grant read/write access to Google Drive specifically.
You can also use `https://www.googleapis.com/auth/drive.readonly` for read only access.
- Click "Authorise"
##### 3. Configure rclone, assuming a new install
@ -277,20 +277,16 @@ y/n> # Auto config, n
##### 4. Verify that it's working
- `rclone -v --drive-impersonate foo@example.com lsf gdrive:backup`
- The arguments do:
- `-v` - verbose logging
- `--drive-impersonate foo@example.com` - this is what does
the magic, pretending to be user foo.
- `lsf` - list files in a parsing friendly way
- `gdrive:backup` - use the remote called gdrive, work in
the folder named backup.
- `rclone -v --drive-impersonate foo@example.com lsf gdrive:backup`
- The arguments do: - `-v` - verbose logging - `--drive-impersonate foo@example.com` - this is what does
the magic, pretending to be user foo. - `lsf` - list files in a parsing friendly way - `gdrive:backup` - use the remote called gdrive, work in
the folder named backup.
Note: in case you configured a specific root folder on gdrive and rclone is unable to access the contents of that folder when using `--drive-impersonate`, do this instead:
- in the gdrive web interface, share your root folder with the user/email of the new Service Account you created/selected at step 1
- use rclone without specifying the `--drive-impersonate` option, like this:
`rclone -v lsf gdrive:backup`
- in the gdrive web interface, share your root folder with the user/email of the new Service Account you created/selected at step 1
- use rclone without specifying the `--drive-impersonate` option, like this:
`rclone -v lsf gdrive:backup`
### Shared drives (team drives)
@ -342,12 +338,15 @@ It does this by combining multiple `list` calls into a single API request.
This works by combining many `'%s' in parents` filters into one expression.
To list the contents of directories a, b and c, the following requests will be send by the regular `List` function:
```
trashed=false and 'a' in parents
trashed=false and 'b' in parents
trashed=false and 'c' in parents
```
These can now be combined into a single request:
```
trashed=false and ('a' in parents or 'b' in parents or 'c' in parents)
```
@ -357,6 +356,7 @@ It will use the `--checkers` value to specify the number of requests to run in
In tests, these batch requests were up to 20x faster than the regular method.
Running the following command against different sized folders gives:
```
rclone lsjson -vv -R --checkers=6 gdrive:folder
```
@ -396,8 +396,8 @@ revision of that file.
Revisions follow the standard google policy which at time of writing
was
* They are deleted after 30 days or 100 revisions (whatever comes first).
* They do not count towards a user storage quota.
- They are deleted after 30 days or 100 revisions (whatever comes first).
- They do not count towards a user storage quota.
### Deleting files
@ -527,7 +527,7 @@ This list can be changed by Google Drive at any time and might not
represent the currently available conversions.
| Extension | Mime Type | Description |
| --------- |-----------| ------------|
| --------- | ------------------------------------------------------------------------- | ---------------------------------------- |
| bmp | image/bmp | Windows Bitmap format |
| csv | text/csv | Standard CSV format for Spreadsheets |
| doc | application/msword | Classic Word file |
@ -543,7 +543,7 @@ represent the currently available conversions.
| odt | application/vnd.oasis.opendocument.text | Openoffice Document |
| pdf | application/pdf | Adobe PDF Format |
| pjpeg | image/pjpeg | Progressive JPEG Image |
| png | image/png | PNG Image Format|
| png | image/png | PNG Image Format |
| pptx | application/vnd.openxmlformats-officedocument.presentationml.presentation | Microsoft Office Powerpoint |
| rtf | application/rtf | Rich Text Format |
| svg | image/svg+xml | Scalable Vector Graphics Format |
@ -561,13 +561,14 @@ when opened. The link file extension has to be specified as a
Google Documents.
| Extension | Description | OS Support |
| --------- | ----------- | ---------- |
| --------- | --------------------------------------- | -------------- |
| desktop | freedesktop.org specified desktop entry | Linux |
| link.html | An HTML Document with a redirect | All |
| url | INI style link file | macOS, Windows |
| webloc | macOS specific XML format | macOS |
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/drive/drive.go then run make backenddocs" >}}
### Standard options
Here are the Standard options specific to drive (Google Drive).
@ -701,7 +702,6 @@ Leave blank normally.
Fill in to access "Computers" folders (see docs), or for rclone to use
a non root folder as its starting point.
Properties:
- Config: root_folder_id
@ -806,7 +806,6 @@ in this mode.
Do **not** use this flag when trying to download Google Docs - rclone
will fail to download them.
Properties:
- Config: show_all_gdocs
@ -1137,8 +1136,6 @@ be removed.
See: https://github.com/rclone/rclone/issues/3631
Properties:
- Config: disable_http2
@ -1161,7 +1158,6 @@ Google don't document so it may break in the future.
See: https://github.com/rclone/rclone/issues/3857
Properties:
- Config: stop_on_upload_limit
@ -1182,7 +1178,6 @@ the in-progress sync.
Note that this detection is relying on error message strings which
Google don't document so it may break in the future.
Properties:
- Config: stop_on_download_limit
@ -1198,7 +1193,6 @@ Normally rclone dereferences shortcut files making them appear as if
they are the original file (see [the shortcuts section](#shortcuts)).
If this flag is set then rclone will ignore shortcut files completely.
Properties:
- Config: skip_shortcuts
@ -1212,7 +1206,6 @@ If set skip dangling shortcut files.
If this is set then rclone will not show any dangling shortcuts in listings.
Properties:
- Config: skip_dangling_shortcuts
@ -1240,7 +1233,6 @@ Note also that opening the folder once in the web interface (with the
user you've authenticated rclone with) seems to be enough so that the
resource key is not needed.
Properties:
- Config: resource_key
@ -1268,7 +1260,6 @@ This flag allows the work-around to be disabled. This is **not**
recommended in normal use - only if you have a particular case you are
having trouble with like many empty directories.
Properties:
- Config: fast_list_bug_fix
@ -1288,7 +1279,6 @@ ownership will generate an email to the new owner (this can't be
disabled), and you can't transfer ownership to someone outside your
organization.
Properties:
- Config: metadata_owner
@ -1318,7 +1308,6 @@ Note that rclone drops any inherited permissions on Shared Drives and
any owner permission on My Drives as these are duplicated in the owner
metadata.
Properties:
- Config: metadata_permissions
@ -1355,7 +1344,6 @@ from two different accounts you will have to create the labels in
advance and use the metadata mapper to translate the IDs between the
two accounts.
Properties:
- Config: metadata_labels
@ -1425,7 +1413,7 @@ Metadata is supported on files and directories.
Here are the possible system metadata items for the drive backend.
| Name | Help | Type | Example | Read Only |
|------|------|------|---------|-----------|
| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | ----------------------------- | --------- |
| btime | Time of file birth (creation) with mS accuracy. Note that this is only writable on fresh uploads - it can't be written for updates. | RFC 3339 | 2006-01-02T15:04:05.999Z07:00 | N |
| content-type | The MIME type of the file. | string | text/plain | N |
| copy-requires-writer-permission | Whether the options to copy, print, or download this file, should be disabled for readers and commenters. | boolean | true | N |
@ -1470,7 +1458,6 @@ Usage Examples:
rclone backend get drive: [-o service_account_file] [-o chunk_size]
rclone rc backend/command command=get fs=drive: [-o service_account_file] [-o chunk_size]
Options:
- "chunk_size": show the current upload chunk size
@ -1489,7 +1476,6 @@ Usage Examples:
rclone backend set drive: [-o service_account_file=sa.json] [-o chunk_size=67108864]
rclone rc backend/command command=set fs=drive: [-o service_account_file=sa.json] [-o chunk_size=67108864]
Options:
- "chunk_size": update the current upload chunk size
@ -1518,7 +1504,6 @@ relative to "drive:" to the "destination_shortcut" relative to
"drive2:". This may fail with a permission error if the user
authenticated with "drive2:" can't read files from "drive:".
Options:
- "target": optional target remote for the shortcut destination
@ -1569,11 +1554,10 @@ drives found and a combined drive.
Adding this to the rclone config file will cause those team drives to
be accessible with the aliases shown. Any illegal characters will be
substituted with "_" and duplicate names will have numbers suffixed.
substituted with "\_" and duplicate names will have numbers suffixed.
It will also add a remote called AllDrives which shows all the shared
drives combined into one directory tree.
### untrash
Untrash files and directories
@ -1600,7 +1584,6 @@ Result:
"Errors": 0
}
### copyid
Copy files by ID
@ -1627,6 +1610,31 @@ attempted if possible.
Use the --interactive/-i or --dry-run flag to see what would be copied before copying.
### moveid
Move files by ID
rclone backend moveid remote: [options] [<arguments>+]
This command moves files by ID
Usage:
rclone backend moveid drive: ID path
rclone backend moveid drive: ID1 path1 ID2 path2
It moves the drive file with ID given to the path (an rclone path which
will be passed internally to rclone copyto). The ID and path pairs can be
repeated.
The path should end with a / to indicate copy the file as named to
this directory. If it doesn't end with a / then the last path
component will be used as the file name.
If the destination is a drive backend then server-side moving will be
attempted if possible.
Use the --interactive/-i or --dry-run flag to see what would be moved before moving.
### exportformats
@ -1768,43 +1776,45 @@ It is strongly recommended to use your own client ID as the default rclone ID is
Here is how to create your own Google Drive client ID for rclone:
1. Log into the [Google API
Console](https://console.developers.google.com/) with your Google
account. It doesn't matter what Google account you use. (It need not
be the same account as the Google Drive you want to access)
Console](https://console.developers.google.com/) with your Google
account. It doesn't matter what Google account you use. (It need not
be the same account as the Google Drive you want to access)
2. Select a project or create a new project.
3. Under "ENABLE APIS AND SERVICES" search for "Drive", and enable the
"Google Drive API".
"Google Drive API".
4. Click "Credentials" in the left-side panel (not "Create
credentials", which opens the wizard).
credentials", which opens the wizard).
5. If you already configured an "Oauth Consent Screen", then skip
to the next step; if not, click on "CONFIGURE CONSENT SCREEN" button
(near the top right corner of the right panel), then select "External"
and click on "CREATE"; on the next screen, enter an "Application name"
("rclone" is OK); enter "User Support Email" (your own email is OK);
enter "Developer Contact Email" (your own email is OK); then click on
"Save" (all other data is optional). You will also have to add [some scopes](https://developers.google.com/drive/api/guides/api-specific-auth),
including
- `https://www.googleapis.com/auth/docs`
- `https://www.googleapis.com/auth/drive` in order to be able to edit,
create and delete files with RClone.
- `https://www.googleapis.com/auth/drive.metadata.readonly` which you may also want to add.
- If you want to add all at once, comma separated it would be `https://www.googleapis.com/auth/docs,https://www.googleapis.com/auth/drive,https://www.googleapis.com/auth/drive.metadata.readonly`.
to the next step; if not, click on "CONFIGURE CONSENT SCREEN" button
(near the top right corner of the right panel), then select "External"
and click on "CREATE"; on the next screen, enter an "Application name"
("rclone" is OK); enter "User Support Email" (your own email is OK);
enter "Developer Contact Email" (your own email is OK); then click on
"Save" (all other data is optional). You will also have to add [some scopes](https://developers.google.com/drive/api/guides/api-specific-auth),
including
- `https://www.googleapis.com/auth/docs`
- `https://www.googleapis.com/auth/drive` in order to be able to edit,
create and delete files with RClone.
- `https://www.googleapis.com/auth/drive.metadata.readonly` which you may also want to add.
- If you want to add all at once, comma separated it would be `https://www.googleapis.com/auth/docs,https://www.googleapis.com/auth/drive,https://www.googleapis.com/auth/drive.metadata.readonly`.
6. After adding scopes, click
"Save and continue" to add test users. Be sure to add your own account to
the test users. Once you've added yourself as a test user and saved the
changes, click again on "Credentials" on the left panel to go back to
the "Credentials" screen.
"Save and continue" to add test users. Be sure to add your own account to
the test users. Once you've added yourself as a test user and saved the
changes, click again on "Credentials" on the left panel to go back to
the "Credentials" screen.
(PS: if you are a GSuite user, you could also select "Internal" instead
of "External" above, but this will restrict API use to Google Workspace
users in your organisation).
of "External" above, but this will restrict API use to Google Workspace
users in your organisation).
7. Click on the "+ CREATE CREDENTIALS" button at the top of the screen,
then select "OAuth client ID".
then select "OAuth client ID".
8. Choose an application type of "Desktop app" and click "Create". (the default name is fine)