lib/rest: Calculate correct Content-Length on MultiPart Uploads

This is used in the pcloud backend and in the upcoming 1fichier backend.

backend/pcloud/pcloud.go

@@ -9,11 +9,9 @@ package pcloud
 // FIXME mime type? Fix overview if implement.
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"net/url"
@@ -1091,21 +1089,18 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// opts.Body=0), so upload it as a multpart form POST with
 	// Content-Length set.
 	if size == 0 {
-		formReader, contentType, err := rest.MultipartUpload(in, opts.Parameters, "content", leaf)
+		formReader, contentType, overhead, err := rest.MultipartUpload(in, opts.Parameters, "content", leaf)
 		if err != nil {
 			return errors.Wrap(err, "failed to make multipart upload for 0 length file")
 		}
-		formBody, err := ioutil.ReadAll(formReader)
+
-		if err != nil {
+		contentLength := overhead + size
-			return errors.Wrap(err, "failed to read multipart upload for 0 length file")
-		}
-		length := int64(len(formBody))
 
 		opts.ContentType = contentType
-		opts.Body = bytes.NewBuffer(formBody)
+		opts.Body = formReader
 		opts.Method = "POST"
 		opts.Parameters = nil
-		opts.ContentLength = &length
+		opts.ContentLength = &contentLength
 	}
 
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {

lib/rest/rest.go

@@ -291,12 +291,42 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
 // fileName - is the name of the attached file
 // contentName - the name of the parameter for the file
 //
+// the *int64 returned is the overhead in addition to the file contents, in case Content-Length is required
+//
 // NB This doesn't allow setting the content type of the attachment
-func MultipartUpload(in io.Reader, params url.Values, contentName, fileName string) (io.ReadCloser, string, error) {
+func MultipartUpload(in io.Reader, params url.Values, contentName, fileName string) (io.ReadCloser, string, int64, error) {
 	bodyReader, bodyWriter := io.Pipe()
 	writer := multipart.NewWriter(bodyWriter)
 	contentType := writer.FormDataContentType()
 
+	// Create a Multipart Writer as base for calculating the Content-Length
+	buf := &bytes.Buffer{}
+	dummyMultipartWriter := multipart.NewWriter(buf)
+	err := dummyMultipartWriter.SetBoundary(writer.Boundary())
+	if err != nil {
+		return nil, "", 0, err
+	}
+
+	for key, vals := range params {
+		for _, val := range vals {
+			err := dummyMultipartWriter.WriteField(key, val)
+			if err != nil {
+				return nil, "", 0, err
+			}
+		}
+	}
+	_, err = dummyMultipartWriter.CreateFormFile(contentName, fileName)
+	if err != nil {
+		return nil, "", 0, err
+	}
+
+	err = dummyMultipartWriter.Close()
+	if err != nil {
+		return nil, "", 0, err
+	}
+
+	multipartLength := int64(buf.Len())
+
 	// Pump the data in the background
 	go func() {
 		var err error
@@ -332,7 +362,7 @@ func MultipartUpload(in io.Reader, params url.Values, contentName, fileName stri
 		_ = bodyWriter.Close()
 	}()
 
-	return bodyReader, contentType, nil
+	return bodyReader, contentType, multipartLength, nil
 }
 
 // CallJSON runs Call and decodes the body as a JSON object into response (if not nil)
@@ -403,9 +433,11 @@ func (api *Client) callCodec(opts *Opts, request interface{}, response interface
 			params.Add(opts.MultipartMetadataName, string(requestBody))
 		}
 		opts = opts.Copy()
-		opts.Body, opts.ContentType, err = MultipartUpload(opts.Body, params, opts.MultipartContentName, opts.MultipartFileName)
+
-		if err != nil {
+		var overhead int64
-			return nil, err
+		opts.Body, opts.ContentType, overhead, err = MultipartUpload(opts.Body, params, opts.MultipartContentName, opts.MultipartFileName)
+		if opts.ContentLength != nil {
+			*opts.ContentLength += overhead
 		}
 	}
 	resp, err = api.Call(opts)