copyurl: add --stdout flag to write to stdout

cmd/copyurl/copyurl.go

@@ -2,6 +2,8 @@ package copyurl
 
 import (
 	"context"
+	"errors"
+	"os"
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs"
@@ -12,37 +14,55 @@ import (
 
 var (
 	autoFilename = false
+	stdout       = false
 )
 
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
-	flags.BoolVarP(cmdFlags, &autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the url and use it for destination file path")
+	flags.BoolVarP(cmdFlags, &autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the URL and use it for destination file path")
+	flags.BoolVarP(cmdFlags, &stdout, "stdout", "", stdout, "Write the output to stdout rather than a file")
 }
 
 var commandDefinition = &cobra.Command{
 	Use:   "copyurl https://example.com dest:path",
 	Short: `Copy url content to dest.`,
 	Long: `
-Download urls content and copy it to destination 
-without saving it in tmp storage.
+Download a URL's content and copy it to the destination without saving
+it in temporary storage.
 
-Setting --auto-filename flag will cause retrieving file name from url and using it in destination path. 
+Setting --auto-filename will cause the file name to be retreived from
+the from URL (after any redirections) and used in the destination
+path.
+
+Setting --stdout or making the output file name "-" will cause the
+output to be written to standard output.
 `,
-	Run: func(command *cobra.Command, args []string) {
-		cmd.CheckArgs(2, 2, command, args)
+	RunE: func(command *cobra.Command, args []string) (err error) {
+		cmd.CheckArgs(1, 2, command, args)
 
 		var dstFileName string
 		var fsdst fs.Fs
-		if autoFilename {
-			fsdst = cmd.NewFsDir(args[1:])
-		} else {
-			fsdst, dstFileName = cmd.NewFsDstFile(args[1:])
+		if !stdout {
+			if len(args) < 2 {
+				return errors.New("need 2 arguments if not using --stdout")
+			}
+			if args[1] == "-" {
+				stdout = true
+			} else if autoFilename {
+				fsdst = cmd.NewFsDir(args[1:])
+			} else {
+				fsdst, dstFileName = cmd.NewFsDstFile(args[1:])
+			}
 		}
-
 		cmd.Run(true, true, command, func() error {
-			_, err := operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename)
+			if stdout {
+				err = operations.CopyURLToWriter(context.Background(), args[0], os.Stdout)
+			} else {
+				_, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename)
+			}
 			return err
 		})
+		return nil
 	},
 }

fs/operations/operations.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"path"
 	"path/filepath"
 	"sort"
@@ -1616,26 +1617,48 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
 	return obj, nil
 }
 
-// CopyURL copies the data from the url to (fdst, dstFileName)
-func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool) (dst fs.Object, err error) {
+// copyURLFunc is called from CopyURLFn
+type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error)
+
+// copyURLFn copies the data from the url to the function supplied
+func copyURLFn(ctx context.Context, dstFileName string, url string, dstFileNameFromURL bool, fn copyURLFunc) (err error) {
 	client := fshttp.NewClient(fs.Config)
 	resp, err := client.Get(url)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	defer fs.CheckClose(resp.Body, &err)
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return nil, errors.Errorf("CopyURL failed: %s", resp.Status)
+		return errors.Errorf("CopyURL failed: %s", resp.Status)
+	}
+	modTime, err := http.ParseTime(resp.Header.Get("Last-Modified"))
+	if err != nil {
+		modTime = time.Now()
 	}
-
 	if dstFileNameFromURL {
 		dstFileName = path.Base(resp.Request.URL.Path)
 		if dstFileName == "." || dstFileName == "/" {
-			return nil, errors.Errorf("CopyURL failed: file name wasn't found in url")
+			return errors.Errorf("CopyURL failed: file name wasn't found in url")
 		}
 	}
+	return fn(ctx, dstFileName, resp.Body, resp.ContentLength, modTime)
+}
 
-	return RcatSize(ctx, fdst, dstFileName, resp.Body, resp.ContentLength, time.Now())
+// CopyURL copies the data from the url to (fdst, dstFileName)
+func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool) (dst fs.Object, err error) {
+	err = copyURLFn(ctx, dstFileName, url, dstFileNameFromURL, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) {
+		dst, err = RcatSize(ctx, fdst, dstFileName, in, size, modTime)
+		return err
+	})
+	return dst, err
+}
+
+// CopyURLToWriter copies the data from the url to the io.Writer supplied
+func CopyURLToWriter(ctx context.Context, url string, out io.Writer) (err error) {
+	return copyURLFn(ctx, "", url, false, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) {
+		_, err = io.Copy(out, in)
+		return err
+	})
 }
 
 // BackupDir returns the correctly configured --backup-dir

fs/operations/operations_test.go

@@ -665,6 +665,37 @@ func TestCopyURL(t *testing.T) {
 	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported)
 }
 
+func TestCopyURLToWriter(t *testing.T) {
+	contents := "file contents\n"
+
+	// check when reading from regular HTTP server
+	status := 0
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if status != 0 {
+			http.Error(w, "an error ocurred", status)
+			return
+		}
+		_, err := w.Write([]byte(contents))
+		assert.NoError(t, err)
+	})
+	ts := httptest.NewServer(handler)
+	defer ts.Close()
+
+	// test normal fetch
+	var buf bytes.Buffer
+	err := operations.CopyURLToWriter(context.Background(), ts.URL, &buf)
+	require.NoError(t, err)
+	assert.Equal(t, contents, buf.String())
+
+	// test fetch with error
+	status = http.StatusNotFound
+	buf.Reset()
+	err = operations.CopyURLToWriter(context.Background(), ts.URL, &buf)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "Not Found")
+	assert.Equal(t, 0, len(buf.String()))
+}
+
 func TestMoveFile(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()