mirror of
https://github.com/rclone/rclone.git
synced 2024-11-28 11:25:18 +01:00
fs: add --metadata-set flag to specify metadata for uploads
This commit is contained in:
parent
0652ec95db
commit
c4451bc43a
@ -452,7 +452,7 @@ attributes such as file mode, owner, extended attributes (not
|
|||||||
Windows).
|
Windows).
|
||||||
|
|
||||||
Note that arbitrary metadata may be added to objects using the
|
Note that arbitrary metadata may be added to objects using the
|
||||||
`--upload-metadata key=value` flag when the object is first uploaded.
|
`--metadata-set key=value` flag when the object is first uploaded.
|
||||||
This flag can be repeated as many times as necessary.
|
This flag can be repeated as many times as necessary.
|
||||||
|
|
||||||
### Types of metadata
|
### Types of metadata
|
||||||
@ -1332,6 +1332,12 @@ Setting this flag enables rclone to copy the metadata from the source
|
|||||||
to the destination. For local backends this is ownership, permissions,
|
to the destination. For local backends this is ownership, permissions,
|
||||||
xattr etc. See the [#metadata](metadata section) for more info.
|
xattr etc. See the [#metadata](metadata section) for more info.
|
||||||
|
|
||||||
|
### --metadata-set key=value
|
||||||
|
|
||||||
|
Add metadata `key` = `value` when uploading. This can be repeated as
|
||||||
|
many times as required. See the [#metadata](metadata section) for more
|
||||||
|
info.
|
||||||
|
|
||||||
### --cutoff-mode=hard|soft|cautious ###
|
### --cutoff-mode=hard|soft|cautious ###
|
||||||
|
|
||||||
This modifies the behavior of `--max-transfer`
|
This modifies the behavior of `--max-transfer`
|
||||||
|
@ -124,6 +124,7 @@ type ConfigInfo struct {
|
|||||||
UploadHeaders []*HTTPOption
|
UploadHeaders []*HTTPOption
|
||||||
DownloadHeaders []*HTTPOption
|
DownloadHeaders []*HTTPOption
|
||||||
Headers []*HTTPOption
|
Headers []*HTTPOption
|
||||||
|
MetadataSet Metadata // extra metadata to write when uploading
|
||||||
RefreshTimes bool
|
RefreshTimes bool
|
||||||
NoConsole bool
|
NoConsole bool
|
||||||
TrafficClass uint8
|
TrafficClass uint8
|
||||||
|
@ -37,6 +37,7 @@ var (
|
|||||||
uploadHeaders []string
|
uploadHeaders []string
|
||||||
downloadHeaders []string
|
downloadHeaders []string
|
||||||
headers []string
|
headers []string
|
||||||
|
metadataSet []string
|
||||||
)
|
)
|
||||||
|
|
||||||
// AddFlags adds the non filing system specific flags to the command
|
// AddFlags adds the non filing system specific flags to the command
|
||||||
@ -129,6 +130,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
|
|||||||
flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions")
|
flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions")
|
||||||
flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions")
|
flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions")
|
||||||
flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions")
|
flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions")
|
||||||
|
flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading")
|
||||||
flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files")
|
flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files")
|
||||||
flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)")
|
flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)")
|
||||||
flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21")
|
flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21")
|
||||||
@ -271,6 +273,20 @@ func SetFlags(ci *fs.ConfigInfo) {
|
|||||||
if len(headers) != 0 {
|
if len(headers) != 0 {
|
||||||
ci.Headers = ParseHeaders(headers)
|
ci.Headers = ParseHeaders(headers)
|
||||||
}
|
}
|
||||||
|
if len(headers) != 0 {
|
||||||
|
ci.Headers = ParseHeaders(headers)
|
||||||
|
}
|
||||||
|
if len(metadataSet) != 0 {
|
||||||
|
ci.MetadataSet = make(fs.Metadata, len(metadataSet))
|
||||||
|
for _, kv := range metadataSet {
|
||||||
|
equal := strings.IndexRune(kv, '=')
|
||||||
|
if equal < 0 {
|
||||||
|
log.Fatalf("Failed to parse '%s' as metadata key=value.", kv)
|
||||||
|
}
|
||||||
|
ci.MetadataSet[strings.ToLower(kv[:equal])] = kv[equal+1:]
|
||||||
|
}
|
||||||
|
fs.Debugf(nil, "MetadataUpload %v", ci.MetadataSet)
|
||||||
|
}
|
||||||
if len(dscp) != 0 {
|
if len(dscp) != 0 {
|
||||||
if value, ok := parseDSCP(dscp); ok {
|
if value, ok := parseDSCP(dscp); ok {
|
||||||
ci.TrafficClass = value << 2
|
ci.TrafficClass = value << 2
|
||||||
|
@ -31,6 +31,30 @@ func (m *Metadata) Set(k, v string) {
|
|||||||
(*m)[k] = v
|
(*m)[k] = v
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Merge other into m
|
||||||
|
//
|
||||||
|
// If m is nil, then it will get made
|
||||||
|
func (m *Metadata) Merge(other Metadata) {
|
||||||
|
for k, v := range other {
|
||||||
|
if *m == nil {
|
||||||
|
*m = make(Metadata, len(other))
|
||||||
|
}
|
||||||
|
(*m)[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MergeOptions gets any Metadata from the options passed in and
|
||||||
|
// stores it in m (which may be nil).
|
||||||
|
//
|
||||||
|
// If there is no m then metadata will be nil
|
||||||
|
func (m *Metadata) MergeOptions(options []OpenOption) {
|
||||||
|
for _, opt := range options {
|
||||||
|
if metadataOption, ok := opt.(MetadataOption); ok {
|
||||||
|
m.Merge(Metadata(metadataOption))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// GetMetadata from an ObjectInfo
|
// GetMetadata from an ObjectInfo
|
||||||
//
|
//
|
||||||
// If the object has no metadata then metadata will be nil
|
// If the object has no metadata then metadata will be nil
|
||||||
@ -41,3 +65,15 @@ func GetMetadata(ctx context.Context, o ObjectInfo) (metadata Metadata, err erro
|
|||||||
}
|
}
|
||||||
return do.Metadata(ctx)
|
return do.Metadata(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetMetadataOptions from an ObjectInfo and merge it with any in options
|
||||||
|
//
|
||||||
|
// If the object has no metadata then metadata will be nil
|
||||||
|
func GetMetadataOptions(ctx context.Context, o ObjectInfo, options []OpenOption) (metadata Metadata, err error) {
|
||||||
|
metadata, err = GetMetadata(ctx, o)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
metadata.MergeOptions(options)
|
||||||
|
return metadata, nil
|
||||||
|
}
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
package fs
|
package fs
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@ -15,3 +16,81 @@ func TestMetadataSet(t *testing.T) {
|
|||||||
m.Set("key", "value2")
|
m.Set("key", "value2")
|
||||||
assert.Equal(t, "value2", m["key"])
|
assert.Equal(t, "value2", m["key"])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMetadataMerge(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
in Metadata
|
||||||
|
merge Metadata
|
||||||
|
want Metadata
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
in: Metadata{},
|
||||||
|
merge: Metadata{},
|
||||||
|
want: Metadata{},
|
||||||
|
}, {
|
||||||
|
in: nil,
|
||||||
|
merge: nil,
|
||||||
|
want: nil,
|
||||||
|
}, {
|
||||||
|
in: nil,
|
||||||
|
merge: Metadata{},
|
||||||
|
want: nil,
|
||||||
|
}, {
|
||||||
|
in: nil,
|
||||||
|
merge: Metadata{"a": "1", "b": "2"},
|
||||||
|
want: Metadata{"a": "1", "b": "2"},
|
||||||
|
}, {
|
||||||
|
in: Metadata{"a": "1", "b": "2"},
|
||||||
|
merge: nil,
|
||||||
|
want: Metadata{"a": "1", "b": "2"},
|
||||||
|
}, {
|
||||||
|
in: Metadata{"a": "1", "b": "2"},
|
||||||
|
merge: Metadata{"b": "B", "c": "3"},
|
||||||
|
want: Metadata{"a": "1", "b": "B", "c": "3"},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
what := fmt.Sprintf("in=%v, merge=%v", test.in, test.merge)
|
||||||
|
test.in.Merge(test.merge)
|
||||||
|
assert.Equal(t, test.want, test.in, what)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetadataMergeOptions(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
in Metadata
|
||||||
|
opts []OpenOption
|
||||||
|
want Metadata
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
opts: []OpenOption{},
|
||||||
|
want: nil,
|
||||||
|
}, {
|
||||||
|
opts: []OpenOption{&HTTPOption{}},
|
||||||
|
want: nil,
|
||||||
|
}, {
|
||||||
|
opts: []OpenOption{MetadataOption{"a": "1", "b": "2"}},
|
||||||
|
want: Metadata{"a": "1", "b": "2"},
|
||||||
|
}, {
|
||||||
|
opts: []OpenOption{
|
||||||
|
&HTTPOption{},
|
||||||
|
MetadataOption{"a": "1", "b": "2"},
|
||||||
|
MetadataOption{"b": "B", "c": "3"},
|
||||||
|
&HTTPOption{},
|
||||||
|
},
|
||||||
|
want: Metadata{"a": "1", "b": "B", "c": "3"},
|
||||||
|
}, {
|
||||||
|
in: Metadata{"a": "first", "z": "OK"},
|
||||||
|
opts: []OpenOption{
|
||||||
|
&HTTPOption{},
|
||||||
|
MetadataOption{"a": "1", "b": "2"},
|
||||||
|
MetadataOption{"b": "B", "c": "3"},
|
||||||
|
&HTTPOption{},
|
||||||
|
},
|
||||||
|
want: Metadata{"a": "1", "b": "B", "c": "3", "z": "OK"},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
what := fmt.Sprintf("in=%v, opts=%v", test.in, test.opts)
|
||||||
|
test.in.MergeOptions(test.opts)
|
||||||
|
assert.Equal(t, test.want, test.in, what)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -258,6 +258,24 @@ func (o NullOption) Mandatory() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MetadataOption defines an Option which does nothing
|
||||||
|
type MetadataOption Metadata
|
||||||
|
|
||||||
|
// Header formats the option as an http header
|
||||||
|
func (o MetadataOption) Header() (key string, value string) {
|
||||||
|
return "", ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// String formats the option into human-readable form
|
||||||
|
func (o MetadataOption) String() string {
|
||||||
|
return fmt.Sprintf("MetadataOption(%v)", Metadata(o))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mandatory returns whether the option must be parsed or can be ignored
|
||||||
|
func (o MetadataOption) Mandatory() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// OpenOptionAddHeaders adds each header found in options to the
|
// OpenOptionAddHeaders adds each header found in options to the
|
||||||
// headers map provided the key was non empty.
|
// headers map provided the key was non empty.
|
||||||
func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {
|
func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) {
|
||||||
|
@ -132,6 +132,16 @@ func TestNullOption(t *testing.T) {
|
|||||||
assert.Equal(t, false, opt.Mandatory())
|
assert.Equal(t, false, opt.Mandatory())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMetadataOption(t *testing.T) {
|
||||||
|
opt := MetadataOption{"onion": "ice cream"}
|
||||||
|
var _ OpenOption = opt // check interface
|
||||||
|
assert.Equal(t, "MetadataOption(map[onion:ice cream])", opt.String())
|
||||||
|
key, value := opt.Header()
|
||||||
|
assert.Equal(t, "", key)
|
||||||
|
assert.Equal(t, "", value)
|
||||||
|
assert.Equal(t, false, opt.Mandatory())
|
||||||
|
}
|
||||||
|
|
||||||
func TestFixRangeOptions(t *testing.T) {
|
func TestFixRangeOptions(t *testing.T) {
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
name string
|
name string
|
||||||
|
@ -489,6 +489,9 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
|
|||||||
for _, option := range ci.UploadHeaders {
|
for _, option := range ci.UploadHeaders {
|
||||||
options = append(options, option)
|
options = append(options, option)
|
||||||
}
|
}
|
||||||
|
if ci.MetadataSet != nil {
|
||||||
|
options = append(options, fs.MetadataOption(ci.MetadataSet))
|
||||||
|
}
|
||||||
if doUpdate {
|
if doUpdate {
|
||||||
actionTaken = "Copied (replaced existing)"
|
actionTaken = "Copied (replaced existing)"
|
||||||
err = dst.Update(ctx, in, wrappedSrc, options...)
|
err = dst.Update(ctx, in, wrappedSrc, options...)
|
||||||
@ -1381,6 +1384,9 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
|
|||||||
for _, option := range ci.UploadHeaders {
|
for _, option := range ci.UploadHeaders {
|
||||||
options = append(options, option)
|
options = append(options, option)
|
||||||
}
|
}
|
||||||
|
if ci.MetadataSet != nil {
|
||||||
|
options = append(options, fs.MetadataOption(ci.MetadataSet))
|
||||||
|
}
|
||||||
|
|
||||||
compare := func(dst fs.Object) error {
|
compare := func(dst fs.Object) error {
|
||||||
var sums map[hash.Type]string
|
var sums map[hash.Type]string
|
||||||
|
Loading…
Reference in New Issue
Block a user