fs: add --metadata-set flag to specify metadata for uploads

This commit is contained in:
Nick Craig-Wood 2022-05-24 15:46:07 +01:00
parent 0652ec95db
commit c4451bc43a
8 changed files with 173 additions and 1 deletions

View File

@ -452,7 +452,7 @@ attributes such as file mode, owner, extended attributes (not
Windows).
Note that arbitrary metadata may be added to objects using the
`--upload-metadata key=value` flag when the object is first uploaded.
`--metadata-set key=value` flag when the object is first uploaded.
This flag can be repeated as many times as necessary.
### Types of metadata
@ -1332,6 +1332,12 @@ Setting this flag enables rclone to copy the metadata from the source
to the destination. For local backends this is ownership, permissions,
xattr etc. See the [#metadata](metadata section) for more info.
### --metadata-set key=value
Add metadata `key` = `value` when uploading. This can be repeated as
many times as required. See the [#metadata](metadata section) for more
info.
### --cutoff-mode=hard|soft|cautious ###
This modifies the behavior of `--max-transfer`

View File

@ -124,6 +124,7 @@ type ConfigInfo struct {
UploadHeaders []*HTTPOption
DownloadHeaders []*HTTPOption
Headers []*HTTPOption
MetadataSet Metadata // extra metadata to write when uploading
RefreshTimes bool
NoConsole bool
TrafficClass uint8

View File

@ -37,6 +37,7 @@ var (
uploadHeaders []string
downloadHeaders []string
headers []string
metadataSet []string
)
// AddFlags adds the non filing system specific flags to the command
@ -129,6 +130,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions")
flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions")
flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions")
flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading")
flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files")
flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)")
flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21")
@ -271,6 +273,20 @@ func SetFlags(ci *fs.ConfigInfo) {
if len(headers) != 0 {
ci.Headers = ParseHeaders(headers)
}
if len(headers) != 0 {
ci.Headers = ParseHeaders(headers)
}
if len(metadataSet) != 0 {
ci.MetadataSet = make(fs.Metadata, len(metadataSet))
for _, kv := range metadataSet {
equal := strings.IndexRune(kv, '=')
if equal < 0 {
log.Fatalf("Failed to parse '%s' as metadata key=value.", kv)
}
ci.MetadataSet[strings.ToLower(kv[:equal])] = kv[equal+1:]
}
fs.Debugf(nil, "MetadataUpload %v", ci.MetadataSet)
}
if len(dscp) != 0 {
if value, ok := parseDSCP(dscp); ok {
ci.TrafficClass = value << 2

View File

@ -31,6 +31,30 @@ func (m *Metadata) Set(k, v string) {
(*m)[k] = v
}
// Merge other into m
//
// If m is nil, then it will get made
func (m *Metadata) Merge(other Metadata) {
for k, v := range other {
if *m == nil {
*m = make(Metadata, len(other))
}
(*m)[k] = v
}
}
// MergeOptions gets any Metadata from the options passed in and
// stores it in m (which may be nil).
//
// If there is no m then metadata will be nil
func (m *Metadata) MergeOptions(options []OpenOption) {
for _, opt := range options {
if metadataOption, ok := opt.(MetadataOption); ok {
m.Merge(Metadata(metadataOption))
}
}
}
// GetMetadata from an ObjectInfo
//
// If the object has no metadata then metadata will be nil
@ -41,3 +65,15 @@ func GetMetadata(ctx context.Context, o ObjectInfo) (metadata Metadata, err erro
}
return do.Metadata(ctx)
}
// GetMetadataOptions from an ObjectInfo and merge it with any in options
//
// If the object has no metadata then metadata will be nil
func GetMetadataOptions(ctx context.Context, o ObjectInfo, options []OpenOption) (metadata Metadata, err error) {
metadata, err = GetMetadata(ctx, o)
if err != nil {
return nil, err
}
metadata.MergeOptions(options)
return metadata, nil
}

View File

@ -1,6 +1,7 @@
package fs
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
@ -15,3 +16,81 @@ func TestMetadataSet(t *testing.T) {
m.Set("key", "value2")
assert.Equal(t, "value2", m["key"])
}
func TestMetadataMerge(t *testing.T) {
for _, test := range []struct {
in Metadata
merge Metadata
want Metadata
}{
{
in: Metadata{},
merge: Metadata{},
want: Metadata{},
}, {
in: nil,
merge: nil,
want: nil,
}, {
in: nil,
merge: Metadata{},
want: nil,
}, {
in: nil,
merge: Metadata{"a": "1", "b": "2"},
want: Metadata{"a": "1", "b": "2"},
}, {
in: Metadata{"a": "1", "b": "2"},
merge: nil,
want: Metadata{"a": "1", "b": "2"},
}, {
in: Metadata{"a": "1", "b": "2"},
merge: Metadata{"b": "B", "c": "3"},
want: Metadata{"a": "1", "b": "B", "c": "3"},
},
} {
what := fmt.Sprintf("in=%v, merge=%v", test.in, test.merge)
test.in.Merge(test.merge)
assert.Equal(t, test.want, test.in, what)
}
}
func TestMetadataMergeOptions(t *testing.T) {
for _, test := range []struct {
in Metadata
opts []OpenOption
want Metadata
}{
{
opts: []OpenOption{},
want: nil,
}, {
opts: []OpenOption{&HTTPOption{}},
want: nil,
}, {
opts: []OpenOption{MetadataOption{"a": "1", "b": "2"}},
want: Metadata{"a": "1", "b": "2"},
}, {
opts: []OpenOption{
&HTTPOption{},
MetadataOption{"a": "1", "b": "2"},
MetadataOption{"b": "B", "c": "3"},
&HTTPOption{},
},
want: Metadata{"a": "1", "b": "B", "c": "3"},
}, {
in: Metadata{"a": "first", "z": "OK"},
opts: []OpenOption{
&HTTPOption{},
MetadataOption{"a": "1", "b": "2"},
MetadataOption{"b": "B", "c": "3"},
&HTTPOption{},
},
want: Metadata{"a": "1", "b": "B", "c": "3", "z": "OK"},
},
} {
what := fmt.Sprintf("in=%v, opts=%v", test.in, test.opts)
test.in.MergeOptions(test.opts)
assert.Equal(t, test.want, test.in, what)
}
}

View File

@ -258,6 +258,24 @@ func (o NullOption) Mandatory() bool {
return false
}
// MetadataOption defines an Option which does nothing
type MetadataOption Metadata
// Header formats the option as an http header
func (o MetadataOption) Header() (key string, value string) {
return "", ""
}
// String formats the option into human-readable form
func (o MetadataOption) String() string {
return fmt.Sprintf("MetadataOption(%v)", Metadata(o))
}
// Mandatory returns whether the option must be parsed or can be ignored
func (o MetadataOption) Mandatory() bool {
return false
}
// OpenOptionAddHeaders adds each header found in options to the
// headers map provided the key was non empty.
func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {

View File

@ -132,6 +132,16 @@ func TestNullOption(t *testing.T) {
assert.Equal(t, false, opt.Mandatory())
}
func TestMetadataOption(t *testing.T) {
opt := MetadataOption{"onion": "ice cream"}
var _ OpenOption = opt // check interface
assert.Equal(t, "MetadataOption(map[onion:ice cream])", opt.String())
key, value := opt.Header()
assert.Equal(t, "", key)
assert.Equal(t, "", value)
assert.Equal(t, false, opt.Mandatory())
}
func TestFixRangeOptions(t *testing.T) {
for _, test := range []struct {
name string

View File

@ -489,6 +489,9 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
for _, option := range ci.UploadHeaders {
options = append(options, option)
}
if ci.MetadataSet != nil {
options = append(options, fs.MetadataOption(ci.MetadataSet))
}
if doUpdate {
actionTaken = "Copied (replaced existing)"
err = dst.Update(ctx, in, wrappedSrc, options...)
@ -1381,6 +1384,9 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
for _, option := range ci.UploadHeaders {
options = append(options, option)
}
if ci.MetadataSet != nil {
options = append(options, fs.MetadataOption(ci.MetadataSet))
}
compare := func(dst fs.Object) error {
var sums map[hash.Type]string