extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-12-22 15:11:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Make ContentType be preserved for cloud -> cloud copies - fixes #733

Browse Source
This commit is contained in:
Nick Craig-Wood 2016-09-21 22:13:24 +01:00
parent 6c9a258d82
commit 945f49ab5e
28 changed files with 265 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
amazonclouddrive/amazonclouddrive.go
View File

@ -846,6 +846,14 @@ func (o *Object) Remove() error {
return err
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType() string {
if o.info.ContentProperties.ContentType != nil {
return *o.info.ContentProperties.ContentType
}
return ""
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
@ -853,5 +861,6 @@ var (
// _ fs.Copier = (*Fs)(nil)
// _ fs.Mover = (*Fs)(nil)
// _ fs.DirMover = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = &Object{}
)

1
amazonclouddrive/amazonclouddrive_test.go
View File

@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }

28
b2/b2.go
View File

@ -98,12 +98,13 @@ type Fs struct {
// Object describes a b2 object
type Object struct {
fs *Fs // what this object is part of
remote string // The remote path
id string // b2 id of the file
modTime time.Time // The modified time of the object if known
sha1 string // SHA-1 hash if known
size int64 // Size of the object
fs *Fs // what this object is part of
remote string // The remote path
id string // b2 id of the file
modTime time.Time // The modified time of the object if known
sha1 string // SHA-1 hash if known
size int64 // Size of the object
mimeType string // Content-Type of the object
}
// ------------------------------------------------------------
@ -896,9 +897,10 @@ func (o *Object) Size() int64 {
// o.modTime
// o.size
// o.sha1
func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp api.Timestamp, Info map[string]string) (err error) {
func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp api.Timestamp, Info map[string]string, mimeType string) (err error) {
o.id = ID
o.sha1 = SHA1
o.mimeType = mimeType
// Read SHA1 from metadata if it exists and isn't set
if o.sha1 == "" || o.sha1 == "none" {
o.sha1 = Info[sha1Key]
@ -917,7 +919,7 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
// o.size
// o.sha1
func (o *Object) decodeMetaData(info *api.File) (err error) {
return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info)
return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType)
}
// decodeMetaDataFileInfo sets the metadata in the object from an api.FileInfo
@ -928,7 +930,7 @@ func (o *Object) decodeMetaData(info *api.File) (err error) {
// o.size
// o.sha1
func (o *Object) decodeMetaDataFileInfo(info *api.FileInfo) (err error) {
return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info)
return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType)
}
// readMetaData gets the metadata if it hasn't already been fetched
@ -1285,7 +1287,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) (err error) {
ExtraHeaders: map[string]string{
"Authorization": upload.AuthorizationToken,
"X-Bz-File-Name": urlEncode(o.fs.root + o.remote),
"Content-Type": fs.MimeType(o),
"Content-Type": fs.MimeType(src),
sha1Header: calculatedSha1,
timeHeader: timeString(modTime),
},
@ -1337,10 +1339,16 @@ func (o *Object) Remove() error {
return nil
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType() string {
return o.mimeType
}
// Check the interfaces are satisfied
var (
_ fs.Fs = &Fs{}
_ fs.Purger = &Fs{}
_ fs.CleanUpper = &Fs{}
_ fs.Object = &Object{}
_ fs.MimeTyper = &Object{}
)

1
b2/b2_test.go
View File

@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }

1
crypt/crypt2_test.go
View File

@ -47,6 +47,7 @@ func TestObjectFs2(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote2(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes2(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime2(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType2(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime2(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize2(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen2(t *testing.T) { fstests.TestObjectOpen(t) }

1
crypt/crypt_test.go
View File

@ -47,6 +47,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }

43
docs/content/overview.md
View File

@ -15,19 +15,19 @@ show through.
Here is an overview of the major features of each cloud storage system.
| Name | Hash | ModTime | Case Insensitive | Duplicate Files |
| ---------------------- |:-------:|:-------:|:----------------:|:---------------:|
| Google Drive | MD5 | Yes | No | Yes |
| Amazon S3 | MD5 | Yes | No | No |
| Openstack Swift | MD5 | Yes | No | No |
| Dropbox | - | No | Yes | No |
| Google Cloud Storage | MD5 | Yes | No | No |
| Amazon Drive | MD5 | No | Yes | No |
| Microsoft One Drive | SHA1 | Yes | Yes | No |
| Hubic | MD5 | Yes | No | No |
| Backblaze B2 | SHA1 | Yes | No | No |
| Yandex Disk | MD5 | Yes | No | No |
| The local filesystem | All | Yes | Depends | No |
| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type |
| ---------------------- |:-------:|:-------:|:----------------:|:---------------:|:---------:|
| Google Drive | MD5 | Yes | No | Yes | R/W |
| Amazon S3 | MD5 | Yes | No | No | R/W |
| Openstack Swift | MD5 | Yes | No | No | R/W |
| Dropbox | - | No | Yes | No | R |
| Google Cloud Storage | MD5 | Yes | No | No | R/W |
| Amazon Drive | MD5 | No | Yes | No | R |
| Microsoft One Drive | SHA1 | Yes | Yes | No | R |
| Hubic | MD5 | Yes | No | No | R/W |
| Backblaze B2 | SHA1 | Yes | No | No | R/W |
| Yandex Disk | MD5 | Yes | No | No | R/W |
| The local filesystem | All | Yes | Depends | No | - |
### Hash ###
@ -78,6 +78,23 @@ objects with the same name.
This confuses rclone greatly when syncing - use the `rclone dedupe`
command to rename or remove duplicates.
### MIME Type ###
MIME types (also known as media types) classify types of documents
using a simple text classification, eg `text/html` or
`application/pdf`.
Some cloud storage systems support reading (`R`) the MIME type of
objects and some support writing (`W`) the MIME type of objects.
The MIME type can be important if you are serving files directly to
HTTP from the storage system.
If you are copying from a remote which supports reading (`R`) to a
remote which supports writing (`W`) then rclone will preserve the MIME
types. Otherwise they will be guessed from the extension, or the
remote itself may assign the MIME type.
## Optional Features ##
All the remotes support a basic set of features, but there are some

19
drive/drive.go
View File

@ -137,6 +137,7 @@ type Object struct {
bytes int64 // size of the object
modifiedDate string // RFC3339 time it was last modified
isDocument bool // if set this is a Google doc
mimeType string
}
// ------------------------------------------------------------
@ -533,7 +534,7 @@ func (f *Fs) createFileInfo(remote string, modTime time.Time, size int64) (*Obje
Title: leaf,
Description: leaf,
Parents: []*drive.ParentReference{{Id: directoryID}},
MimeType: fs.MimeType(o),
MimeType: fs.MimeTypeFromName(remote),
ModifiedDate: modTime.Format(timeFormatOut),
}
return o, createInfo, nil
@ -845,6 +846,7 @@ func (o *Object) setMetaData(info *drive.File) {
o.md5sum = strings.ToLower(info.Md5Checksum)
o.bytes = info.FileSize
o.modifiedDate = info.ModifiedDate
o.mimeType = info.MimeType
}
// readMetaData gets the info if it hasn't already been fetched
@ -1009,7 +1011,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
}
updateInfo := &drive.File{
Id: o.id,
MimeType: fs.MimeType(o),
MimeType: fs.MimeType(src),
ModifiedDate: modTime.Format(timeFormatOut),
}
@ -1027,7 +1029,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
}
} else {
// Upload the file in chunks
info, err = o.fs.Upload(in, size, fs.MimeType(o), updateInfo, o.remote)
info, err = o.fs.Upload(in, size, updateInfo.MimeType, updateInfo, o.remote)
if err != nil {
return err
}
@ -1053,6 +1055,16 @@ func (o *Object) Remove() error {
return err
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType() string {
err := o.readMetaData()
if err != nil {
fs.Log(o, "Failed to read metadata: %v", err)
return ""
}
return o.mimeType
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
@ -1062,4 +1074,5 @@ var (
_ fs.DirMover = (*Fs)(nil)
_ fs.PutUncheckeder = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = &Object{}
)

1
drive/drive_test.go
View File

@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }

25
dropbox/dropbox.go
View File

@ -110,6 +110,7 @@ type Object struct {
bytes int64 // size of the object
modTime time.Time // time it was last modified
hasMetadata bool // metadata is valid
mimeType string // content type according to the server
}
// ------------------------------------------------------------
@ -622,6 +623,7 @@ func (o *Object) Size() int64 {
func (o *Object) setMetadataFromEntry(info *dropbox.Entry) {
o.bytes = info.Bytes
o.modTime = time.Time(info.ClientMtime)
o.mimeType = info.MimeType
o.hasMetadata = true
}
@ -745,12 +747,23 @@ func (o *Object) Remove() error {
return err
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType() string {
err := o.readMetaData()
if err != nil {
fs.Log(o, "Failed to read metadata: %v", err)
return ""
}
return o.mimeType
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
_ fs.Copier = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.Fs = (*Fs)(nil)
_ fs.Copier = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = (*Object)(nil)
)

1
dropbox/dropbox_test.go
View File

@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }

7
fs/fs.go
View File

@ -210,6 +210,13 @@ type BasicInfo interface {
Size() int64
}
// MimeTyper is an optional interface for Object
type MimeTyper interface {
// MimeType returns the content type of the Object if
// known, or "" if not
MimeType() string
}
// Purger is an optional interfaces for Fs
type Purger interface {
// Purge all files in the root and the root directory

20
fs/operations.go
View File

@ -169,15 +169,29 @@ func Equal(src, dst Object) bool {
return true
}
// MimeType returns a guess at the mime type from the extension
func MimeType(o ObjectInfo) string {
mimeType := mime.TypeByExtension(path.Ext(o.Remote()))
// MimeTypeFromName returns a guess at the mime type from the name
func MimeTypeFromName(remote string) (mimeType string) {
mimeType = mime.TypeByExtension(path.Ext(remote))
if !strings.ContainsRune(mimeType, '/') {
mimeType = "application/octet-stream"
}
return mimeType
}
// MimeType returns the MimeType from the object, either by calling
// the MimeTyper interface or using MimeTypeFromName
func MimeType(o ObjectInfo) (mimeType string) {
// Read the MimeType from the optional interface if available
if do, ok := o.(MimeTyper); ok {
mimeType = do.MimeType()
Debug(o, "Read MimeType as %q", mimeType)
if mimeType != "" {
return mimeType
}
}
return MimeTypeFromName(o.Remote())
}
// Used to remove a failed copy
//
// Returns whether the file was succesfully removed or not

16
fstest/fstests/fstests.go
View File

@ -501,6 +501,22 @@ func TestObjectModTime(t *testing.T) {
file1.CheckModTime(t, obj, obj.ModTime(), remote.Precision())
}
// TestObjectMimeType tests the MimeType of the object is correct
func TestObjectMimeType(t *testing.T) {
skipIfNotOk(t)
obj := findObject(t, file1.Path)
do, ok := obj.(fs.MimeTyper)
if !ok {
t.Skip("MimeType method not supported")
}
mimeType := do.MimeType()
if strings.ContainsRune(mimeType, ';') {
assert.Equal(t, "text/plain; charset=utf-8", mimeType)
} else {
assert.Equal(t, "text/plain", mimeType)
}
}
// TestObjectSetModTime tests that SetModTime works
func TestObjectSetModTime(t *testing.T) {
skipIfNotOk(t)

28
googlecloudstorage/googlecloudstorage.go
View File

@ -143,12 +143,13 @@ type Fs struct {
//
// Will definitely have info but maybe not meta
type Object struct {
fs *Fs // what this object is part of
remote string // The remote path
url string // download path
md5sum string // The MD5Sum of the object
bytes int64 // Bytes in the object
modTime time.Time // Modified time of the object
fs *Fs // what this object is part of
remote string // The remote path
url string // download path
md5sum string // The MD5Sum of the object
bytes int64 // Bytes in the object
modTime time.Time // Modified time of the object
mimeType string
}
// ------------------------------------------------------------
@ -558,6 +559,7 @@ func (o *Object) Size() int64 {
func (o *Object) setMetaData(info *storage.Object) {
o.url = info.MediaLink
o.bytes = int64(info.Size)
o.mimeType = info.ContentType
// Read md5sum
md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash)
@ -675,7 +677,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
object := storage.Object{
Bucket: o.fs.bucket,
Name: o.fs.root + o.remote,
ContentType: fs.MimeType(o),
ContentType: fs.MimeType(src),
Size: uint64(size),
Updated: modTime.Format(timeFormatOut), // Doesn't get set
Metadata: metadataFromModTime(modTime),
@ -694,9 +696,15 @@ func (o *Object) Remove() error {
return o.fs.svc.Objects.Delete(o.fs.bucket, o.fs.root+o.remote).Do()
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType() string {
return o.mimeType
}
// Check the interfaces are satisfied
var (
_ fs.Fs = &Fs{}
_ fs.Copier = &Fs{}
_ fs.Object = &Object{}
_ fs.Fs = &Fs{}
_ fs.Copier = &Fs{}
_ fs.Object = &Object{}
_ fs.MimeTyper = &Object{}
)

1
googlecloudstorage/googlecloudstorage_test.go
View File

@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }

1
hubic/hubic_test.go
View File

@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }

1
local/local_test.go
View File

@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }

16
onedrive/onedrive.go
View File

@ -98,6 +98,7 @@ type Object struct {
modTime time.Time // modification time of the object
id string // ID of the object
sha1 string // SHA-1 of the object content
mimeType string // Content-Type of object from server (may not be as uploaded)
}
// ------------------------------------------------------------
@ -686,8 +687,11 @@ func (o *Object) setMetaData(info *api.Item) {
// fact uppercase hex strings.
//
// In OneDrive for Business, SHA1 and CRC32 hash values are not returned for files.
if info.File != nil && info.File.Hashes.Sha1Hash != "" {
o.sha1 = strings.ToLower(info.File.Hashes.Sha1Hash)
if info.File != nil {
o.mimeType = info.File.MimeType
if info.File.Hashes.Sha1Hash != "" {
o.sha1 = strings.ToLower(info.File.Hashes.Sha1Hash)
}
}
if info.FileSystemInfo != nil {
o.modTime = time.Time(info.FileSystemInfo.LastModifiedDateTime)
@ -935,6 +939,11 @@ func (o *Object) Remove() error {
return o.fs.deleteObject(o.id)
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType() string {
return o.mimeType
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
@ -942,5 +951,6 @@ var (
_ fs.Copier = (*Fs)(nil)
// _ fs.Mover = (*Fs)(nil)
// _ fs.DirMover = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = &Object{}
)

1
onedrive/onedrive_test.go
View File

@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }

31
s3/s3.go
View File

@ -230,14 +230,15 @@ type Fs struct {
type Object struct {
// Will definitely have everything but meta which may be nil
//
// List will read everything but meta - to fill that in need to call
// readMetaData
// List will read everything but meta & mimeType - to fill
// that in you need to call readMetaData
fs *Fs // what this object is part of
remote string // The remote path
etag string // md5sum of the object
bytes int64 // size of the object
lastModified time.Time // Last modified
meta map[string]*string // The object metadata if known - may be nil
mimeType string // MimeType of object - may be ""
}
// ------------------------------------------------------------
@ -777,6 +778,7 @@ func (o *Object) readMetaData() (err error) {
} else {
o.lastModified = *resp.LastModified
}
o.mimeType = aws.StringValue(resp.ContentType)
return nil
}
@ -818,7 +820,7 @@ func (o *Object) SetModTime(modTime time.Time) error {
}
// Guess the content type
contentType := fs.MimeType(o)
mimeType := fs.MimeType(o)
// Copy the object to itself to update the metadata
key := o.fs.root + o.remote
@ -828,7 +830,7 @@ func (o *Object) SetModTime(modTime time.Time) error {
Bucket: &o.fs.bucket,
ACL: &o.fs.acl,
Key: &key,
ContentType: &contentType,
ContentType: &mimeType,
CopySource: aws.String(url.QueryEscape(sourceKey)),
Metadata: o.meta,
MetadataDirective: &directive,
@ -880,7 +882,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
}
// Guess the content type
contentType := fs.MimeType(o)
mimeType := fs.MimeType(src)
key := o.fs.root + o.remote
req := s3manager.UploadInput{
@ -888,7 +890,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
ACL: &o.fs.acl,
Key: &key,
Body: in,
ContentType: &contentType,
ContentType: &mimeType,
Metadata: metadata,
//ContentLength: &size,
}
@ -920,9 +922,20 @@ func (o *Object) Remove() error {
return err
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType() string {
err := o.readMetaData()
if err != nil {
fs.Log(o, "Failed to read metadata: %v", err)
return ""
}
return o.mimeType
}
// Check the interfaces are satisfied
var (
_ fs.Fs = &Fs{}
_ fs.Copier = &Fs{}
_ fs.Object = &Object{}
_ fs.Fs = &Fs{}
_ fs.Copier = &Fs{}
_ fs.Object = &Object{}
_ fs.MimeTyper = &Object{}
)

1
s3/s3_test.go
View File

@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }

23
swift/swift.go
View File

@ -689,7 +689,7 @@ func urlEncode(str string) string {
// updateChunks updates the existing object using chunks to a separate
// container. It returns a string which prefixes current segments.
func (o *Object) updateChunks(in io.Reader, headers swift.Headers, size int64) (string, error) {
func (o *Object) updateChunks(in io.Reader, headers swift.Headers, size int64, contentType string) (string, error) {
// Create the segmentsContainer if it doesn't exist
err := o.fs.c.ContainerCreate(o.fs.segmentsContainer, nil)
if err != nil {
@ -718,7 +718,7 @@ func (o *Object) updateChunks(in io.Reader, headers swift.Headers, size int64) (
headers["Content-Length"] = "0" // set Content-Length as we know it
emptyReader := bytes.NewReader(nil)
manifestName := o.fs.root + o.remote
_, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", "", headers)
_, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers)
return uniquePrefix + "/", err
}
@ -738,16 +738,17 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
// Set the mtime
m := swift.Metadata{}
m.SetModTime(modTime)
contentType := fs.MimeType(src)
headers := m.ObjectHeaders()
uniquePrefix := ""
if size > int64(chunkSize) {
uniquePrefix, err = o.updateChunks(in, headers, size)
uniquePrefix, err = o.updateChunks(in, headers, size, contentType)
if err != nil {
return err
}
} else {
headers["Content-Length"] = strconv.FormatInt(size, 10) // set Content-Length as we know it
_, err := o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", "", headers)
_, err := o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", contentType, headers)
if err != nil {
return err
}
@ -787,10 +788,16 @@ func (o *Object) Remove() error {
return nil
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType() string {
return o.info.ContentType
}
// Check the interfaces are satisfied
var (
_ fs.Fs = &Fs{}
_ fs.Purger = &Fs{}
_ fs.Copier = &Fs{}
_ fs.Object = &Object{}
_ fs.Fs = &Fs{}
_ fs.Purger = &Fs{}
_ fs.Copier = &Fs{}
_ fs.Object = &Object{}
_ fs.MimeTyper = &Object{}
)

1
swift/swift_test.go
View File

@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }

3
yandex/api/performupload.go
View File

@ -11,11 +11,12 @@ import (
)
// PerformUpload does the actual upload via unscoped PUT request.
func (c *Client) PerformUpload(url string, data io.Reader) (err error) {
func (c *Client) PerformUpload(url string, data io.Reader, contentType string) (err error) {
req, err := http.NewRequest("PUT", url, data)
if err != nil {
return err
}
req.Header.Set("Content-Type", contentType)
//c.setRequestScope(req)

4
yandex/api/upload.go
View File

@ -17,13 +17,13 @@ type UploadResponse struct {
}
// Upload will put specified data to Yandex.Disk.
func (c *Client) Upload(data io.Reader, remotePath string, overwrite bool) error {
func (c *Client) Upload(data io.Reader, remotePath string, overwrite bool, contentType string) error {
ur, err := c.UploadRequest(remotePath, overwrite)
if err != nil {
return err
}
if err := c.PerformUpload(ur.HRef, data); err != nil {
if err := c.PerformUpload(ur.HRef, data, contentType); err != nil {
return err
}

74
yandex/yandex.go
View File

@ -73,11 +73,12 @@ type Fs struct {
// Object describes a swift object
type Object struct {
fs *Fs // what this object is part of
remote string // The remote path
md5sum string // The MD5Sum of the object
bytes uint64 // Bytes in the object
modTime time.Time // Modified time of the object
fs *Fs // what this object is part of
remote string // The remote path
md5sum string // The MD5Sum of the object
bytes uint64 // Bytes in the object
modTime time.Time // Modified time of the object
mimeType string // Content type according to the server
}
// ------------------------------------------------------------
@ -326,31 +327,33 @@ func (f *Fs) newObjectWithInfo(remote string, info *yandex.ResourceInfoResponse)
func (o *Object) setMetaData(info *yandex.ResourceInfoResponse) {
o.bytes = info.Size
o.md5sum = info.Md5
o.mimeType = info.MimeType
if info.CustomProperties["rclone_modified"] == nil {
//read modTime from Modified property of object
t, err := time.Parse(time.RFC3339Nano, info.Modified)
if err != nil {
return
}
o.modTime = t
var modTimeString string
modTimeObj, ok := info.CustomProperties["rclone_modified"]
if ok {
// read modTime from rclone_modified custom_property of object
modTimeString, ok = modTimeObj.(string)
}
if !ok {
// read modTime from Modified property of object as a fallback
modTimeString = info.Modified
}
t, err := time.Parse(time.RFC3339Nano, modTimeString)
if err != nil {
fs.Log("Failed to parse modtime from %q: %v", modTimeString, err)
} else {
// interface{} to string type assertion
if modtimestr, ok := info.CustomProperties["rclone_modified"].(string); ok {
//read modTime from rclone_modified custom_property of object
t, err := time.Parse(time.RFC3339Nano, modtimestr)
if err != nil {
return
}
o.modTime = t
} else {
return //if it is not a string
}
o.modTime = t
}
}
// readMetaData gets the info if it hasn't already been fetched
func (o *Object) readMetaData() (err error) {
// exit if already fetched
if !o.modTime.IsZero() {
return nil
}
//request meta info
var opt2 yandex.ResourceInfoRequestOptions
ResourceInfoResponse, err := o.fs.yd.NewResourceInfoRequest(o.remotePath(), opt2).Exec()
@ -498,8 +501,13 @@ func (o *Object) Remove() error {
// Commits the datastore
func (o *Object) SetModTime(modTime time.Time) error {
remote := o.remotePath()
//set custom_property 'rclone_modified' of object to modTime
return o.fs.yd.SetCustomProperty(remote, "rclone_modified", modTime.Format(time.RFC3339Nano))
// set custom_property 'rclone_modified' of object to modTime
err := o.fs.yd.SetCustomProperty(remote, "rclone_modified", modTime.Format(time.RFC3339Nano))
if err != nil {
return err
}
o.modTime = modTime
return nil
}
// Storable returns whether this object is storable
@ -529,7 +537,8 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
}
//upload file
overwrite := true //overwrite existing file
err := o.fs.yd.Upload(in, remote, overwrite)
mimeType := fs.MimeType(src)
err := o.fs.yd.Upload(in, remote, overwrite, mimeType)
if err == nil {
//if file uploaded sucessfully then return metadata
o.bytes = uint64(size)
@ -587,10 +596,21 @@ func mkDirFullPath(client *yandex.Client, path string) error {
return nil
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType() string {
err := o.readMetaData()
if err != nil {
fs.Log(o, "Failed to read metadata: %v", err)
return ""
}
return o.mimeType
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
//_ fs.Copier = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = &Object{}
)

1
yandex/yandex_test.go
View File

@ -46,6 +46,7 @@ func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }