🕰️ GCS: Compatible with gsutil's mtime metadata

- Write `goog-reserved-file-mtime` in addition to `mtime`.
- Fallback to `goog-reserved-file-mtime` if `mtime` doesn't exist.
- ref rclone/rclone#5331
This commit is contained in:
database64128 2021-05-21 17:11:43 +08:00 committed by Nick Craig-Wood
parent 2188fe38e5
commit ee2fac1855

View File

@ -21,6 +21,7 @@ import (
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"path" "path"
"strconv"
"strings" "strings"
"time" "time"
@ -51,8 +52,9 @@ const (
rcloneClientID = "202264815644.apps.googleusercontent.com" rcloneClientID = "202264815644.apps.googleusercontent.com"
rcloneEncryptedClientSecret = "Uj7C9jGfb9gmeaV70Lh058cNkWvepr-Es9sBm0zdgil7JaOWF1VySw" rcloneEncryptedClientSecret = "Uj7C9jGfb9gmeaV70Lh058cNkWvepr-Es9sBm0zdgil7JaOWF1VySw"
timeFormatIn = time.RFC3339 timeFormatIn = time.RFC3339
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00" timeFormatOut = time.RFC3339Nano
metaMtime = "mtime" // key to store mtime under in metadata metaMtime = "mtime" // key to store mtime in metadata
metaMtimeGsutil = "goog-reserved-file-mtime" // key used by GSUtil to store mtime in metadata
listChunks = 1000 // chunk size to read directory listings listChunks = 1000 // chunk size to read directory listings
minSleep = 10 * time.Millisecond minSleep = 10 * time.Millisecond
) )
@ -928,6 +930,17 @@ func (o *Object) setMetaData(info *storage.Object) {
fs.Debugf(o, "Failed to read mtime from metadata: %s", err) fs.Debugf(o, "Failed to read mtime from metadata: %s", err)
} }
// Fallback to GSUtil mtime
mtimeGsutilString, ok := info.Metadata[metaMtimeGsutil]
if ok {
unixTimeSec, err := strconv.ParseInt(mtimeGsutilString, 10, 64)
if err == nil {
o.modTime = time.Unix(unixTimeSec, 0)
return
}
fs.Debugf(o, "Failed to read GSUtil mtime from metadata: %s", err)
}
// Fallback to the Updated time // Fallback to the Updated time
modTime, err := time.Parse(timeFormatIn, info.Updated) modTime, err := time.Parse(timeFormatIn, info.Updated)
if err != nil { if err != nil {
@ -987,6 +1000,7 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
func metadataFromModTime(modTime time.Time) map[string]string { func metadataFromModTime(modTime time.Time) map[string]string {
metadata := make(map[string]string, 1) metadata := make(map[string]string, 1)
metadata[metaMtime] = modTime.Format(timeFormatOut) metadata[metaMtime] = modTime.Format(timeFormatOut)
metadata[metaMtimeGsutil] = strconv.FormatInt(modTime.Unix(), 10)
return metadata return metadata
} }
@ -998,11 +1012,11 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error)
return err return err
} }
// Add the mtime to the existing metadata // Add the mtime to the existing metadata
mtime := modTime.Format(timeFormatOut)
if object.Metadata == nil { if object.Metadata == nil {
object.Metadata = make(map[string]string, 1) object.Metadata = make(map[string]string, 1)
} }
object.Metadata[metaMtime] = mtime object.Metadata[metaMtime] = modTime.Format(timeFormatOut)
object.Metadata[metaMtimeGsutil] = strconv.FormatInt(modTime.Unix(), 10)
// Copy the object to itself to update the metadata // Copy the object to itself to update the metadata
// Using PATCH requires too many permissions // Using PATCH requires too many permissions
bucket, bucketPath := o.split() bucket, bucketPath := o.split()