mirror of
https://github.com/rclone/rclone.git
synced 2024-11-07 09:04:52 +01:00
fs: add --metadata-set flag to specify metadata for uploads
This commit is contained in:
parent
0652ec95db
commit
c4451bc43a
@ -452,7 +452,7 @@ attributes such as file mode, owner, extended attributes (not
|
||||
Windows).
|
||||
|
||||
Note that arbitrary metadata may be added to objects using the
|
||||
`--upload-metadata key=value` flag when the object is first uploaded.
|
||||
`--metadata-set key=value` flag when the object is first uploaded.
|
||||
This flag can be repeated as many times as necessary.
|
||||
|
||||
### Types of metadata
|
||||
@ -1332,6 +1332,12 @@ Setting this flag enables rclone to copy the metadata from the source
|
||||
to the destination. For local backends this is ownership, permissions,
|
||||
xattr etc. See the [#metadata](metadata section) for more info.
|
||||
|
||||
### --metadata-set key=value
|
||||
|
||||
Add metadata `key` = `value` when uploading. This can be repeated as
|
||||
many times as required. See the [#metadata](metadata section) for more
|
||||
info.
|
||||
|
||||
### --cutoff-mode=hard|soft|cautious ###
|
||||
|
||||
This modifies the behavior of `--max-transfer`
|
||||
|
@ -124,6 +124,7 @@ type ConfigInfo struct {
|
||||
UploadHeaders []*HTTPOption
|
||||
DownloadHeaders []*HTTPOption
|
||||
Headers []*HTTPOption
|
||||
MetadataSet Metadata // extra metadata to write when uploading
|
||||
RefreshTimes bool
|
||||
NoConsole bool
|
||||
TrafficClass uint8
|
||||
|
@ -37,6 +37,7 @@ var (
|
||||
uploadHeaders []string
|
||||
downloadHeaders []string
|
||||
headers []string
|
||||
metadataSet []string
|
||||
)
|
||||
|
||||
// AddFlags adds the non filing system specific flags to the command
|
||||
@ -129,6 +130,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
|
||||
flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions")
|
||||
flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions")
|
||||
flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions")
|
||||
flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading")
|
||||
flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files")
|
||||
flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)")
|
||||
flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21")
|
||||
@ -271,6 +273,20 @@ func SetFlags(ci *fs.ConfigInfo) {
|
||||
if len(headers) != 0 {
|
||||
ci.Headers = ParseHeaders(headers)
|
||||
}
|
||||
if len(headers) != 0 {
|
||||
ci.Headers = ParseHeaders(headers)
|
||||
}
|
||||
if len(metadataSet) != 0 {
|
||||
ci.MetadataSet = make(fs.Metadata, len(metadataSet))
|
||||
for _, kv := range metadataSet {
|
||||
equal := strings.IndexRune(kv, '=')
|
||||
if equal < 0 {
|
||||
log.Fatalf("Failed to parse '%s' as metadata key=value.", kv)
|
||||
}
|
||||
ci.MetadataSet[strings.ToLower(kv[:equal])] = kv[equal+1:]
|
||||
}
|
||||
fs.Debugf(nil, "MetadataUpload %v", ci.MetadataSet)
|
||||
}
|
||||
if len(dscp) != 0 {
|
||||
if value, ok := parseDSCP(dscp); ok {
|
||||
ci.TrafficClass = value << 2
|
||||
|
@ -31,6 +31,30 @@ func (m *Metadata) Set(k, v string) {
|
||||
(*m)[k] = v
|
||||
}
|
||||
|
||||
// Merge other into m
|
||||
//
|
||||
// If m is nil, then it will get made
|
||||
func (m *Metadata) Merge(other Metadata) {
|
||||
for k, v := range other {
|
||||
if *m == nil {
|
||||
*m = make(Metadata, len(other))
|
||||
}
|
||||
(*m)[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
// MergeOptions gets any Metadata from the options passed in and
|
||||
// stores it in m (which may be nil).
|
||||
//
|
||||
// If there is no m then metadata will be nil
|
||||
func (m *Metadata) MergeOptions(options []OpenOption) {
|
||||
for _, opt := range options {
|
||||
if metadataOption, ok := opt.(MetadataOption); ok {
|
||||
m.Merge(Metadata(metadataOption))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetMetadata from an ObjectInfo
|
||||
//
|
||||
// If the object has no metadata then metadata will be nil
|
||||
@ -41,3 +65,15 @@ func GetMetadata(ctx context.Context, o ObjectInfo) (metadata Metadata, err erro
|
||||
}
|
||||
return do.Metadata(ctx)
|
||||
}
|
||||
|
||||
// GetMetadataOptions from an ObjectInfo and merge it with any in options
|
||||
//
|
||||
// If the object has no metadata then metadata will be nil
|
||||
func GetMetadataOptions(ctx context.Context, o ObjectInfo, options []OpenOption) (metadata Metadata, err error) {
|
||||
metadata, err = GetMetadata(ctx, o)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metadata.MergeOptions(options)
|
||||
return metadata, nil
|
||||
}
|
||||
|
@ -1,6 +1,7 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@ -15,3 +16,81 @@ func TestMetadataSet(t *testing.T) {
|
||||
m.Set("key", "value2")
|
||||
assert.Equal(t, "value2", m["key"])
|
||||
}
|
||||
|
||||
func TestMetadataMerge(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in Metadata
|
||||
merge Metadata
|
||||
want Metadata
|
||||
}{
|
||||
{
|
||||
in: Metadata{},
|
||||
merge: Metadata{},
|
||||
want: Metadata{},
|
||||
}, {
|
||||
in: nil,
|
||||
merge: nil,
|
||||
want: nil,
|
||||
}, {
|
||||
in: nil,
|
||||
merge: Metadata{},
|
||||
want: nil,
|
||||
}, {
|
||||
in: nil,
|
||||
merge: Metadata{"a": "1", "b": "2"},
|
||||
want: Metadata{"a": "1", "b": "2"},
|
||||
}, {
|
||||
in: Metadata{"a": "1", "b": "2"},
|
||||
merge: nil,
|
||||
want: Metadata{"a": "1", "b": "2"},
|
||||
}, {
|
||||
in: Metadata{"a": "1", "b": "2"},
|
||||
merge: Metadata{"b": "B", "c": "3"},
|
||||
want: Metadata{"a": "1", "b": "B", "c": "3"},
|
||||
},
|
||||
} {
|
||||
what := fmt.Sprintf("in=%v, merge=%v", test.in, test.merge)
|
||||
test.in.Merge(test.merge)
|
||||
assert.Equal(t, test.want, test.in, what)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataMergeOptions(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in Metadata
|
||||
opts []OpenOption
|
||||
want Metadata
|
||||
}{
|
||||
{
|
||||
opts: []OpenOption{},
|
||||
want: nil,
|
||||
}, {
|
||||
opts: []OpenOption{&HTTPOption{}},
|
||||
want: nil,
|
||||
}, {
|
||||
opts: []OpenOption{MetadataOption{"a": "1", "b": "2"}},
|
||||
want: Metadata{"a": "1", "b": "2"},
|
||||
}, {
|
||||
opts: []OpenOption{
|
||||
&HTTPOption{},
|
||||
MetadataOption{"a": "1", "b": "2"},
|
||||
MetadataOption{"b": "B", "c": "3"},
|
||||
&HTTPOption{},
|
||||
},
|
||||
want: Metadata{"a": "1", "b": "B", "c": "3"},
|
||||
}, {
|
||||
in: Metadata{"a": "first", "z": "OK"},
|
||||
opts: []OpenOption{
|
||||
&HTTPOption{},
|
||||
MetadataOption{"a": "1", "b": "2"},
|
||||
MetadataOption{"b": "B", "c": "3"},
|
||||
&HTTPOption{},
|
||||
},
|
||||
want: Metadata{"a": "1", "b": "B", "c": "3", "z": "OK"},
|
||||
},
|
||||
} {
|
||||
what := fmt.Sprintf("in=%v, opts=%v", test.in, test.opts)
|
||||
test.in.MergeOptions(test.opts)
|
||||
assert.Equal(t, test.want, test.in, what)
|
||||
}
|
||||
}
|
||||
|
@ -258,6 +258,24 @@ func (o NullOption) Mandatory() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// MetadataOption defines an Option which does nothing
|
||||
type MetadataOption Metadata
|
||||
|
||||
// Header formats the option as an http header
|
||||
func (o MetadataOption) Header() (key string, value string) {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// String formats the option into human-readable form
|
||||
func (o MetadataOption) String() string {
|
||||
return fmt.Sprintf("MetadataOption(%v)", Metadata(o))
|
||||
}
|
||||
|
||||
// Mandatory returns whether the option must be parsed or can be ignored
|
||||
func (o MetadataOption) Mandatory() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// OpenOptionAddHeaders adds each header found in options to the
|
||||
// headers map provided the key was non empty.
|
||||
func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {
|
||||
|
@ -132,6 +132,16 @@ func TestNullOption(t *testing.T) {
|
||||
assert.Equal(t, false, opt.Mandatory())
|
||||
}
|
||||
|
||||
func TestMetadataOption(t *testing.T) {
|
||||
opt := MetadataOption{"onion": "ice cream"}
|
||||
var _ OpenOption = opt // check interface
|
||||
assert.Equal(t, "MetadataOption(map[onion:ice cream])", opt.String())
|
||||
key, value := opt.Header()
|
||||
assert.Equal(t, "", key)
|
||||
assert.Equal(t, "", value)
|
||||
assert.Equal(t, false, opt.Mandatory())
|
||||
}
|
||||
|
||||
func TestFixRangeOptions(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
|
@ -489,6 +489,9 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
|
||||
for _, option := range ci.UploadHeaders {
|
||||
options = append(options, option)
|
||||
}
|
||||
if ci.MetadataSet != nil {
|
||||
options = append(options, fs.MetadataOption(ci.MetadataSet))
|
||||
}
|
||||
if doUpdate {
|
||||
actionTaken = "Copied (replaced existing)"
|
||||
err = dst.Update(ctx, in, wrappedSrc, options...)
|
||||
@ -1381,6 +1384,9 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
|
||||
for _, option := range ci.UploadHeaders {
|
||||
options = append(options, option)
|
||||
}
|
||||
if ci.MetadataSet != nil {
|
||||
options = append(options, fs.MetadataOption(ci.MetadataSet))
|
||||
}
|
||||
|
||||
compare := func(dst fs.Object) error {
|
||||
var sums map[hash.Type]string
|
||||
|
Loading…
Reference in New Issue
Block a user