fs: add --metadata-set flag to specify metadata for uploads

docs/content/docs.md

@@ -452,7 +452,7 @@ attributes such as file mode, owner, extended attributes (not
 Windows).
 
 Note that arbitrary metadata may be added to objects using the
-`--upload-metadata key=value` flag when the object is first uploaded.
+`--metadata-set key=value` flag when the object is first uploaded.
 This flag can be repeated as many times as necessary.
 
 ### Types of metadata
@@ -1332,6 +1332,12 @@ Setting this flag enables rclone to copy the metadata from the source
 to the destination. For local backends this is ownership, permissions,
 xattr etc. See the [#metadata](metadata section) for more info.
 
+### --metadata-set key=value
+
+Add metadata `key` = `value` when uploading. This can be repeated as
+many times as required. See the [#metadata](metadata section) for more
+info.
+
 ### --cutoff-mode=hard|soft|cautious ###
 
 This modifies the behavior of `--max-transfer`

fs/config.go

@@ -124,6 +124,7 @@ type ConfigInfo struct {
 	UploadHeaders          []*HTTPOption
 	DownloadHeaders        []*HTTPOption
 	Headers                []*HTTPOption
+	MetadataSet            Metadata // extra metadata to write when uploading
 	RefreshTimes           bool
 	NoConsole              bool
 	TrafficClass           uint8

fs/config/configflags/configflags.go

@@ -37,6 +37,7 @@ var (
 	uploadHeaders   []string
 	downloadHeaders []string
 	headers         []string
+	metadataSet     []string
 )
 
 // AddFlags adds the non filing system specific flags to the command
@@ -129,6 +130,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
 	flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions")
 	flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions")
 	flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions")
+	flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading")
 	flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files")
 	flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)")
 	flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21")
@@ -271,6 +273,20 @@ func SetFlags(ci *fs.ConfigInfo) {
 	if len(headers) != 0 {
 		ci.Headers = ParseHeaders(headers)
 	}
+	if len(headers) != 0 {
+		ci.Headers = ParseHeaders(headers)
+	}
+	if len(metadataSet) != 0 {
+		ci.MetadataSet = make(fs.Metadata, len(metadataSet))
+		for _, kv := range metadataSet {
+			equal := strings.IndexRune(kv, '=')
+			if equal < 0 {
+				log.Fatalf("Failed to parse '%s' as metadata key=value.", kv)
+			}
+			ci.MetadataSet[strings.ToLower(kv[:equal])] = kv[equal+1:]
+		}
+		fs.Debugf(nil, "MetadataUpload %v", ci.MetadataSet)
+	}
 	if len(dscp) != 0 {
 		if value, ok := parseDSCP(dscp); ok {
 			ci.TrafficClass = value << 2

fs/metadata.go

@@ -31,6 +31,30 @@ func (m *Metadata) Set(k, v string) {
 	(*m)[k] = v
 }
 
+// Merge other into m
+//
+// If m is nil, then it will get made
+func (m *Metadata) Merge(other Metadata) {
+	for k, v := range other {
+		if *m == nil {
+			*m = make(Metadata, len(other))
+		}
+		(*m)[k] = v
+	}
+}
+
+// MergeOptions gets any Metadata from the options passed in and
+// stores it in m (which may be nil).
+//
+// If there is no m then metadata will be nil
+func (m *Metadata) MergeOptions(options []OpenOption) {
+	for _, opt := range options {
+		if metadataOption, ok := opt.(MetadataOption); ok {
+			m.Merge(Metadata(metadataOption))
+		}
+	}
+}
+
 // GetMetadata from an ObjectInfo
 //
 // If the object has no metadata then metadata will be nil
@@ -41,3 +65,15 @@ func GetMetadata(ctx context.Context, o ObjectInfo) (metadata Metadata, err erro
 	}
 	return do.Metadata(ctx)
 }
+
+// GetMetadataOptions from an ObjectInfo and merge it with any in options
+//
+// If the object has no metadata then metadata will be nil
+func GetMetadataOptions(ctx context.Context, o ObjectInfo, options []OpenOption) (metadata Metadata, err error) {
+	metadata, err = GetMetadata(ctx, o)
+	if err != nil {
+		return nil, err
+	}
+	metadata.MergeOptions(options)
+	return metadata, nil
+}

fs/metadata_test.go

@@ -1,6 +1,7 @@
 package fs
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -15,3 +16,81 @@ func TestMetadataSet(t *testing.T) {
 	m.Set("key", "value2")
 	assert.Equal(t, "value2", m["key"])
 }
+
+func TestMetadataMerge(t *testing.T) {
+	for _, test := range []struct {
+		in    Metadata
+		merge Metadata
+		want  Metadata
+	}{
+		{
+			in:    Metadata{},
+			merge: Metadata{},
+			want:  Metadata{},
+		}, {
+			in:    nil,
+			merge: nil,
+			want:  nil,
+		}, {
+			in:    nil,
+			merge: Metadata{},
+			want:  nil,
+		}, {
+			in:    nil,
+			merge: Metadata{"a": "1", "b": "2"},
+			want:  Metadata{"a": "1", "b": "2"},
+		}, {
+			in:    Metadata{"a": "1", "b": "2"},
+			merge: nil,
+			want:  Metadata{"a": "1", "b": "2"},
+		}, {
+			in:    Metadata{"a": "1", "b": "2"},
+			merge: Metadata{"b": "B", "c": "3"},
+			want:  Metadata{"a": "1", "b": "B", "c": "3"},
+		},
+	} {
+		what := fmt.Sprintf("in=%v, merge=%v", test.in, test.merge)
+		test.in.Merge(test.merge)
+		assert.Equal(t, test.want, test.in, what)
+	}
+}
+
+func TestMetadataMergeOptions(t *testing.T) {
+	for _, test := range []struct {
+		in   Metadata
+		opts []OpenOption
+		want Metadata
+	}{
+		{
+			opts: []OpenOption{},
+			want: nil,
+		}, {
+			opts: []OpenOption{&HTTPOption{}},
+			want: nil,
+		}, {
+			opts: []OpenOption{MetadataOption{"a": "1", "b": "2"}},
+			want: Metadata{"a": "1", "b": "2"},
+		}, {
+			opts: []OpenOption{
+				&HTTPOption{},
+				MetadataOption{"a": "1", "b": "2"},
+				MetadataOption{"b": "B", "c": "3"},
+				&HTTPOption{},
+			},
+			want: Metadata{"a": "1", "b": "B", "c": "3"},
+		}, {
+			in: Metadata{"a": "first", "z": "OK"},
+			opts: []OpenOption{
+				&HTTPOption{},
+				MetadataOption{"a": "1", "b": "2"},
+				MetadataOption{"b": "B", "c": "3"},
+				&HTTPOption{},
+			},
+			want: Metadata{"a": "1", "b": "B", "c": "3", "z": "OK"},
+		},
+	} {
+		what := fmt.Sprintf("in=%v, opts=%v", test.in, test.opts)
+		test.in.MergeOptions(test.opts)
+		assert.Equal(t, test.want, test.in, what)
+	}
+}

fs/open_options.go

@@ -258,6 +258,24 @@ func (o NullOption) Mandatory() bool {
 	return false
 }
 
+// MetadataOption defines an Option which does nothing
+type MetadataOption Metadata
+
+// Header formats the option as an http header
+func (o MetadataOption) Header() (key string, value string) {
+	return "", ""
+}
+
+// String formats the option into human-readable form
+func (o MetadataOption) String() string {
+	return fmt.Sprintf("MetadataOption(%v)", Metadata(o))
+}
+
+// Mandatory returns whether the option must be parsed or can be ignored
+func (o MetadataOption) Mandatory() bool {
+	return false
+}
+
 // OpenOptionAddHeaders adds each header found in options to the
 // headers map provided the key was non empty.
 func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {

fs/open_options_test.go

@@ -132,6 +132,16 @@ func TestNullOption(t *testing.T) {
 	assert.Equal(t, false, opt.Mandatory())
 }
 
+func TestMetadataOption(t *testing.T) {
+	opt := MetadataOption{"onion": "ice cream"}
+	var _ OpenOption = opt // check interface
+	assert.Equal(t, "MetadataOption(map[onion:ice cream])", opt.String())
+	key, value := opt.Header()
+	assert.Equal(t, "", key)
+	assert.Equal(t, "", value)
+	assert.Equal(t, false, opt.Mandatory())
+}
+
 func TestFixRangeOptions(t *testing.T) {
 	for _, test := range []struct {
 		name string

fs/operations/operations.go

@@ -489,6 +489,9 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 						for _, option := range ci.UploadHeaders {
 							options = append(options, option)
 						}
+						if ci.MetadataSet != nil {
+							options = append(options, fs.MetadataOption(ci.MetadataSet))
+						}
 						if doUpdate {
 							actionTaken = "Copied (replaced existing)"
 							err = dst.Update(ctx, in, wrappedSrc, options...)
@@ -1381,6 +1384,9 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
 	for _, option := range ci.UploadHeaders {
 		options = append(options, option)
 	}
+	if ci.MetadataSet != nil {
+		options = append(options, fs.MetadataOption(ci.MetadataSet))
+	}
 
 	compare := func(dst fs.Object) error {
 		var sums map[hash.Type]string