copyurl: add --auto-filename flag for using file name from url in destination path (#3451)

cmd/copyurl/copyurl.go

@@ -4,12 +4,18 @@ import (
 	"context"
 
 	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/spf13/cobra"
 )
 
+var (
+	autoFilename = false
+)
+
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
+	commandDefintion.Flags().BoolVarP(&autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the url and use it for destination file path")
 }
 
 var commandDefintion = &cobra.Command{
@@ -18,13 +24,22 @@ var commandDefintion = &cobra.Command{
 	Long: `
 Download urls content and copy it to destination 
 without saving it in tmp storage.
+
+Setting --auto-filename flag will cause retrieving file name from url and using it in destination path. 
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
-		fsdst, dstFileName := cmd.NewFsDstFile(args[1:])
+
+		var dstFileName string
+		var fsdst fs.Fs
+		if autoFilename {
+			fsdst = cmd.NewFsDir(args[1:])
+		} else {
+			fsdst, dstFileName = cmd.NewFsDstFile(args[1:])
+		}
 
 		cmd.Run(true, true, command, func() error {
-			_, err := operations.CopyURL(context.Background(), fsdst, dstFileName, args[0])
+			_, err := operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename)
 			return err
 		})
 	},

docs/content/commands/rclone_copyurl.md

@@ -22,7 +22,8 @@ rclone copyurl https://example.com dest:path [flags]
 ### Options
 
 ```
-  -h, --help   help for copyurl
+  -a, --auto-filename   Get the file name from the url and use it for destination file path
+  -h, --help            help for copyurl
 ```
 
 See the [global flags page](/flags/) for global options not listed here.

fs/operations/operations.go

@@ -1586,7 +1586,7 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
 }
 
 // CopyURL copies the data from the url to (fdst, dstFileName)
-func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string) (dst fs.Object, err error) {
+func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool) (dst fs.Object, err error) {
 	client := fshttp.NewClient(fs.Config)
 	resp, err := client.Get(url)
 	if err != nil {
@@ -1596,6 +1596,14 @@ func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string) (d
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		return nil, errors.Errorf("CopyURL failed: %s", resp.Status)
 	}
+
+	if dstFileNameFromURL {
+		dstFileName = path.Base(resp.Request.URL.Path)
+		if dstFileName == "." || dstFileName == "/" {
+			return nil, errors.Errorf("CopyURL failed: file name wasn't found in url")
+		}
+	}
+
 	return RcatSize(ctx, fdst, dstFileName, resp.Body, resp.ContentLength, time.Now())
 }
 

fs/operations/operations_test.go

@@ -706,15 +706,27 @@ func TestCopyURL(t *testing.T) {
 	ts := httptest.NewServer(handler)
 	defer ts.Close()
 
-	o, err := operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL)
+	o, err := operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false)
 	require.NoError(t, err)
 	assert.Equal(t, int64(len(contents)), o.Size())
 
 	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported)
 
+	// Check auto file naming
+	status = 0
+	urlFileName := "filename.txt"
+	o, err = operations.CopyURL(context.Background(), r.Fremote, "", ts.URL+"/"+urlFileName, true)
+	require.NoError(t, err)
+	assert.Equal(t, int64(len(contents)), o.Size())
+	assert.Equal(t, urlFileName, o.Remote())
+
+	// Check auto file naming when url without file name
+	o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, true)
+	require.Error(t, err)
+
 	// Check an error is returned for a 404
 	status = http.StatusNotFound
-	o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL)
+	o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "Not Found")
 	assert.Nil(t, o)
@@ -730,10 +742,10 @@ func TestCopyURL(t *testing.T) {
 	tss := httptest.NewTLSServer(handler)
 	defer tss.Close()
 
-	o, err = operations.CopyURL(context.Background(), r.Fremote, "file2", tss.URL)
+	o, err = operations.CopyURL(context.Background(), r.Fremote, "file2", tss.URL, false)
 	require.NoError(t, err)
 	assert.Equal(t, int64(len(contents)), o.Size())
-	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported)
 }
 
 func TestMoveFile(t *testing.T) {

fs/operations/rc.go

@@ -149,7 +149,7 @@ func init() {
 		{name: "rmdirs", title: "Remove all the empty directories in the path", help: "- leaveRoot - boolean, set to true not to delete the root\n"},
 		{name: "delete", title: "Remove files in the path", noRemote: true},
 		{name: "deletefile", title: "Remove the single file pointed to"},
-		{name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n"},
+		{name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n - autoFilename - boolean, set to true to retrieve destination file name from url"},
 		{name: "cleanup", title: "Remove trashed files in the remote or path", noRemote: true},
 	} {
 		op := op
@@ -214,7 +214,9 @@ func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bo
 		if err != nil {
 			return nil, err
 		}
-		_, err = CopyURL(ctx, f, remote, url)
+		autoFilename, _ := in.GetBool("autoFilename")
+
+		_, err = CopyURL(ctx, f, remote, url, autoFilename)
 		return nil, err
 	case "cleanup":
 		return nil, CleanUp(ctx, f)

fs/operations/rc_test.go

@@ -105,15 +105,37 @@ func TestRcCopyurl(t *testing.T) {
 	defer ts.Close()
 
 	in := rc.Params{
-		"fs":     r.FremoteName,
-		"remote": "file1",
-		"url":    ts.URL,
+		"fs":           r.FremoteName,
+		"remote":       "file1",
+		"url":          ts.URL,
+		"autoFilename": false,
 	}
 	out, err := call.Fn(context.Background(), in)
 	require.NoError(t, err)
 	assert.Equal(t, rc.Params(nil), out)
 
-	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported)
+	urlFileName := "filename.txt"
+	in = rc.Params{
+		"fs":           r.FremoteName,
+		"remote":       "",
+		"url":          ts.URL + "/" + urlFileName,
+		"autoFilename": true,
+	}
+	out, err = call.Fn(context.Background(), in)
+	require.NoError(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	in = rc.Params{
+		"fs":           r.FremoteName,
+		"remote":       "",
+		"url":          ts.URL,
+		"autoFilename": true,
+	}
+	out, err = call.Fn(context.Background(), in)
+	require.Error(t, err)
+	assert.Equal(t, rc.Params(nil), out)
+
+	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported)
 }
 
 // operations/delete: Remove files in the path