cmd: add copyurl command - Fixes #1320

cmd/all/all.go

@@ -14,6 +14,7 @@ import (
 	_ "github.com/ncw/rclone/cmd/config"
 	_ "github.com/ncw/rclone/cmd/copy"
 	_ "github.com/ncw/rclone/cmd/copyto"
+	_ "github.com/ncw/rclone/cmd/copyurl"
 	_ "github.com/ncw/rclone/cmd/cryptcheck"
 	_ "github.com/ncw/rclone/cmd/cryptdecode"
 	_ "github.com/ncw/rclone/cmd/dbhashsum"

cmd/copyurl/copyurl.go

@@ -0,0 +1,39 @@
+package copyurl
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/fs/operations"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefintion)
+}
+
+var commandDefintion = &cobra.Command{
+	Use:   "copyurl https://example.com dest:path",
+	Short: `Copy url content to dest.`,
+	Long: `
+Download urls content and copy it to destination 
+without saving it in tmp storage.
+`,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(2, 2, command, args)
+		fsdst, dstFileName := cmd.NewFsDstFile(args[1:])
+
+		cmd.Run(true, true, command, func() error {
+			resp, err := http.Get(args[0])
+			if err != nil {
+
+				return err
+			}
+
+			_, err = operations.RcatSize(fsdst, dstFileName, resp.Body, resp.ContentLength, time.Now())
+
+			return err
+		})
+	},
+}

cmd/serve/restic/restic.go

@@ -6,13 +6,13 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
 	"regexp"
 	"strconv"
 	"strings"
+
 	"time"
 
 	"github.com/ncw/rclone/cmd"
@@ -21,7 +21,6 @@ import (
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/accounting"
 	"github.com/ncw/rclone/fs/fserrors"
-	"github.com/ncw/rclone/fs/object"
 	"github.com/ncw/rclone/fs/operations"
 	"github.com/ncw/rclone/fs/walk"
 	"github.com/spf13/cobra"
@@ -326,46 +325,18 @@ func (s *server) postObject(w http.ResponseWriter, r *http.Request, remote strin
 		if err == nil {
 			fs.Errorf(remote, "Post request: file already exists, refusing to overwrite in append-only mode")
 			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
+
 			return
 		}
 	}
 
-	// fs.Debugf(s.f, "content length = %d", r.ContentLength)
-	if r.ContentLength >= 0 {
-		// Size known use Put
-		accounting.Stats.Transferring(remote)
-		body := ioutil.NopCloser(r.Body)                                   // we let the server close the body
-		in := accounting.NewAccountSizeName(body, r.ContentLength, remote) // account the transfer (no buffering)
-		var err error
-		defer func() {
-			closeErr := in.Close()
-			if closeErr != nil {
-				fs.Errorf(remote, "Post request: close failed: %v", closeErr)
-				if err == nil {
-					err = closeErr
-				}
-			}
-			ok := err == nil
-			accounting.Stats.DoneTransferring(remote, err == nil)
-			if !ok {
-				accounting.Stats.Error(err)
-			}
-		}()
-		info := object.NewStaticObjectInfo(remote, time.Now(), r.ContentLength, true, nil, s.f)
-		_, err = s.f.Put(in, info)
-		if err != nil {
-			fs.Errorf(remote, "Post request put error: %v", err)
-			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
-			return
-		}
-	} else {
-		// Size unknown use Rcat
-		_, err := operations.Rcat(s.f, remote, r.Body, time.Now())
-		if err != nil {
-			fs.Errorf(remote, "Post request rcat error: %v", err)
-			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
-			return
-		}
+	_, err := operations.RcatSize(s.f, remote, r.Body, r.ContentLength, time.Now())
+	if err != nil {
+		accounting.Stats.Error(err)
+		fs.Errorf(remote, "Post request rcat error: %v", err)
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+
+		return
 	}
 }
 

fs/operations/operations.go

@@ -1315,6 +1315,45 @@ func NeedTransfer(dst, src fs.Object) bool {
 	return true
 }
 
+// RcatSize reads data from the Reader until EOF and uploads it to a file on remote.
+// Pass in size >=0 if known, <0 if not known
+func RcatSize(fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (dst fs.Object, err error) {
+	var obj fs.Object
+
+	if size >= 0 {
+		// Size known use Put
+		accounting.Stats.Transferring(dstFileName)
+		body := ioutil.NopCloser(in)                                 // we let the server close the body
+		in := accounting.NewAccountSizeName(body, size, dstFileName) // account the transfer (no buffering)
+		var err error
+		defer func() {
+			closeErr := in.Close()
+			if closeErr != nil {
+				accounting.Stats.Error(closeErr)
+				fs.Errorf(dstFileName, "Post request: close failed: %v", closeErr)
+			}
+			accounting.Stats.DoneTransferring(dstFileName, err == nil)
+		}()
+		info := object.NewStaticObjectInfo(dstFileName, modTime, size, true, nil, fdst)
+		obj, err = fdst.Put(in, info)
+		if err != nil {
+			fs.Errorf(dstFileName, "Post request put error: %v", err)
+
+			return nil, err
+		}
+	} else {
+		// Size unknown use Rcat
+		obj, err = Rcat(fdst, dstFileName, in, modTime)
+		if err != nil {
+			fs.Errorf(dstFileName, "Post request rcat error: %v", err)
+
+			return nil, err
+		}
+	}
+
+	return obj, nil
+}
+
 // moveOrCopyFile moves or copies a single file possibly to a new name
 func moveOrCopyFile(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) {
 	dstFilePath := path.Join(fdst.Root(), dstFileName)

fs/operations/operations_test.go

@@ -468,6 +468,32 @@ func TestRmdirsLeaveRoot(t *testing.T) {
 	)
 }
 
+func TestRcatSize(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	const body = "------------------------------------------------------------"
+	file1 := r.WriteFile("potato1", body, t1)
+	file2 := r.WriteFile("potato2", body, t2)
+	// Test with known length
+	bodyReader := ioutil.NopCloser(strings.NewReader(body))
+	obj, err := operations.RcatSize(r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime)
+	require.NoError(t, err)
+	assert.Equal(t, int64(len(body)), obj.Size())
+	assert.Equal(t, file1.Path, obj.Remote())
+
+	// Test with unknown length
+	bodyReader = ioutil.NopCloser(strings.NewReader(body)) // reset Reader
+	ioutil.NopCloser(strings.NewReader(body))
+	obj, err = operations.RcatSize(r.Fremote, file2.Path, bodyReader, -1, file2.ModTime)
+	require.NoError(t, err)
+	assert.Equal(t, int64(len(body)), obj.Size())
+	assert.Equal(t, file2.Path, obj.Remote())
+
+	// Check files exist
+	fstest.CheckItems(t, r.Fremote, file1, file2)
+}
+
 func TestMoveFile(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()