fs: add --metadata-set flag to specify metadata for uploads

This commit is contained in:
Nick Craig-Wood 2022-05-24 15:46:07 +01:00
parent 0652ec95db
commit c4451bc43a
8 changed files with 173 additions and 1 deletions

View File

@ -452,7 +452,7 @@ attributes such as file mode, owner, extended attributes (not
Windows). Windows).
Note that arbitrary metadata may be added to objects using the Note that arbitrary metadata may be added to objects using the
`--upload-metadata key=value` flag when the object is first uploaded. `--metadata-set key=value` flag when the object is first uploaded.
This flag can be repeated as many times as necessary. This flag can be repeated as many times as necessary.
### Types of metadata ### Types of metadata
@ -1332,6 +1332,12 @@ Setting this flag enables rclone to copy the metadata from the source
to the destination. For local backends this is ownership, permissions, to the destination. For local backends this is ownership, permissions,
xattr etc. See the [#metadata](metadata section) for more info. xattr etc. See the [#metadata](metadata section) for more info.
### --metadata-set key=value
Add metadata `key` = `value` when uploading. This can be repeated as
many times as required. See the [#metadata](metadata section) for more
info.
### --cutoff-mode=hard|soft|cautious ### ### --cutoff-mode=hard|soft|cautious ###
This modifies the behavior of `--max-transfer` This modifies the behavior of `--max-transfer`

View File

@ -124,6 +124,7 @@ type ConfigInfo struct {
UploadHeaders []*HTTPOption UploadHeaders []*HTTPOption
DownloadHeaders []*HTTPOption DownloadHeaders []*HTTPOption
Headers []*HTTPOption Headers []*HTTPOption
MetadataSet Metadata // extra metadata to write when uploading
RefreshTimes bool RefreshTimes bool
NoConsole bool NoConsole bool
TrafficClass uint8 TrafficClass uint8

View File

@ -37,6 +37,7 @@ var (
uploadHeaders []string uploadHeaders []string
downloadHeaders []string downloadHeaders []string
headers []string headers []string
metadataSet []string
) )
// AddFlags adds the non filing system specific flags to the command // AddFlags adds the non filing system specific flags to the command
@ -129,6 +130,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions") flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions")
flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions") flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions")
flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions") flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions")
flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading")
flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files") flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files")
flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)") flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)")
flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21") flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21")
@ -271,6 +273,20 @@ func SetFlags(ci *fs.ConfigInfo) {
if len(headers) != 0 { if len(headers) != 0 {
ci.Headers = ParseHeaders(headers) ci.Headers = ParseHeaders(headers)
} }
if len(headers) != 0 {
ci.Headers = ParseHeaders(headers)
}
if len(metadataSet) != 0 {
ci.MetadataSet = make(fs.Metadata, len(metadataSet))
for _, kv := range metadataSet {
equal := strings.IndexRune(kv, '=')
if equal < 0 {
log.Fatalf("Failed to parse '%s' as metadata key=value.", kv)
}
ci.MetadataSet[strings.ToLower(kv[:equal])] = kv[equal+1:]
}
fs.Debugf(nil, "MetadataUpload %v", ci.MetadataSet)
}
if len(dscp) != 0 { if len(dscp) != 0 {
if value, ok := parseDSCP(dscp); ok { if value, ok := parseDSCP(dscp); ok {
ci.TrafficClass = value << 2 ci.TrafficClass = value << 2

View File

@ -31,6 +31,30 @@ func (m *Metadata) Set(k, v string) {
(*m)[k] = v (*m)[k] = v
} }
// Merge other into m
//
// If m is nil, then it will get made
func (m *Metadata) Merge(other Metadata) {
for k, v := range other {
if *m == nil {
*m = make(Metadata, len(other))
}
(*m)[k] = v
}
}
// MergeOptions gets any Metadata from the options passed in and
// stores it in m (which may be nil).
//
// If there is no m then metadata will be nil
func (m *Metadata) MergeOptions(options []OpenOption) {
for _, opt := range options {
if metadataOption, ok := opt.(MetadataOption); ok {
m.Merge(Metadata(metadataOption))
}
}
}
// GetMetadata from an ObjectInfo // GetMetadata from an ObjectInfo
// //
// If the object has no metadata then metadata will be nil // If the object has no metadata then metadata will be nil
@ -41,3 +65,15 @@ func GetMetadata(ctx context.Context, o ObjectInfo) (metadata Metadata, err erro
} }
return do.Metadata(ctx) return do.Metadata(ctx)
} }
// GetMetadataOptions from an ObjectInfo and merge it with any in options
//
// If the object has no metadata then metadata will be nil
func GetMetadataOptions(ctx context.Context, o ObjectInfo, options []OpenOption) (metadata Metadata, err error) {
metadata, err = GetMetadata(ctx, o)
if err != nil {
return nil, err
}
metadata.MergeOptions(options)
return metadata, nil
}

View File

@ -1,6 +1,7 @@
package fs package fs
import ( import (
"fmt"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -15,3 +16,81 @@ func TestMetadataSet(t *testing.T) {
m.Set("key", "value2") m.Set("key", "value2")
assert.Equal(t, "value2", m["key"]) assert.Equal(t, "value2", m["key"])
} }
func TestMetadataMerge(t *testing.T) {
for _, test := range []struct {
in Metadata
merge Metadata
want Metadata
}{
{
in: Metadata{},
merge: Metadata{},
want: Metadata{},
}, {
in: nil,
merge: nil,
want: nil,
}, {
in: nil,
merge: Metadata{},
want: nil,
}, {
in: nil,
merge: Metadata{"a": "1", "b": "2"},
want: Metadata{"a": "1", "b": "2"},
}, {
in: Metadata{"a": "1", "b": "2"},
merge: nil,
want: Metadata{"a": "1", "b": "2"},
}, {
in: Metadata{"a": "1", "b": "2"},
merge: Metadata{"b": "B", "c": "3"},
want: Metadata{"a": "1", "b": "B", "c": "3"},
},
} {
what := fmt.Sprintf("in=%v, merge=%v", test.in, test.merge)
test.in.Merge(test.merge)
assert.Equal(t, test.want, test.in, what)
}
}
func TestMetadataMergeOptions(t *testing.T) {
for _, test := range []struct {
in Metadata
opts []OpenOption
want Metadata
}{
{
opts: []OpenOption{},
want: nil,
}, {
opts: []OpenOption{&HTTPOption{}},
want: nil,
}, {
opts: []OpenOption{MetadataOption{"a": "1", "b": "2"}},
want: Metadata{"a": "1", "b": "2"},
}, {
opts: []OpenOption{
&HTTPOption{},
MetadataOption{"a": "1", "b": "2"},
MetadataOption{"b": "B", "c": "3"},
&HTTPOption{},
},
want: Metadata{"a": "1", "b": "B", "c": "3"},
}, {
in: Metadata{"a": "first", "z": "OK"},
opts: []OpenOption{
&HTTPOption{},
MetadataOption{"a": "1", "b": "2"},
MetadataOption{"b": "B", "c": "3"},
&HTTPOption{},
},
want: Metadata{"a": "1", "b": "B", "c": "3", "z": "OK"},
},
} {
what := fmt.Sprintf("in=%v, opts=%v", test.in, test.opts)
test.in.MergeOptions(test.opts)
assert.Equal(t, test.want, test.in, what)
}
}

View File

@ -258,6 +258,24 @@ func (o NullOption) Mandatory() bool {
return false return false
} }
// MetadataOption defines an Option which does nothing
type MetadataOption Metadata
// Header formats the option as an http header
func (o MetadataOption) Header() (key string, value string) {
return "", ""
}
// String formats the option into human-readable form
func (o MetadataOption) String() string {
return fmt.Sprintf("MetadataOption(%v)", Metadata(o))
}
// Mandatory returns whether the option must be parsed or can be ignored
func (o MetadataOption) Mandatory() bool {
return false
}
// OpenOptionAddHeaders adds each header found in options to the // OpenOptionAddHeaders adds each header found in options to the
// headers map provided the key was non empty. // headers map provided the key was non empty.
func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) { func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {

View File

@ -132,6 +132,16 @@ func TestNullOption(t *testing.T) {
assert.Equal(t, false, opt.Mandatory()) assert.Equal(t, false, opt.Mandatory())
} }
func TestMetadataOption(t *testing.T) {
opt := MetadataOption{"onion": "ice cream"}
var _ OpenOption = opt // check interface
assert.Equal(t, "MetadataOption(map[onion:ice cream])", opt.String())
key, value := opt.Header()
assert.Equal(t, "", key)
assert.Equal(t, "", value)
assert.Equal(t, false, opt.Mandatory())
}
func TestFixRangeOptions(t *testing.T) { func TestFixRangeOptions(t *testing.T) {
for _, test := range []struct { for _, test := range []struct {
name string name string

View File

@ -489,6 +489,9 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
for _, option := range ci.UploadHeaders { for _, option := range ci.UploadHeaders {
options = append(options, option) options = append(options, option)
} }
if ci.MetadataSet != nil {
options = append(options, fs.MetadataOption(ci.MetadataSet))
}
if doUpdate { if doUpdate {
actionTaken = "Copied (replaced existing)" actionTaken = "Copied (replaced existing)"
err = dst.Update(ctx, in, wrappedSrc, options...) err = dst.Update(ctx, in, wrappedSrc, options...)
@ -1381,6 +1384,9 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
for _, option := range ci.UploadHeaders { for _, option := range ci.UploadHeaders {
options = append(options, option) options = append(options, option)
} }
if ci.MetadataSet != nil {
options = append(options, fs.MetadataOption(ci.MetadataSet))
}
compare := func(dst fs.Object) error { compare := func(dst fs.Object) error {
var sums map[hash.Type]string var sums map[hash.Type]string