mirror of
https://github.com/rclone/rclone.git
synced 2024-12-23 15:38:57 +01:00
local: add Metadata support #111
This commit is contained in:
parent
22abd785eb
commit
c556e98f49
@ -42,6 +42,18 @@ func init() {
|
|||||||
Description: "Local Disk",
|
Description: "Local Disk",
|
||||||
NewFs: NewFs,
|
NewFs: NewFs,
|
||||||
CommandHelp: commandHelp,
|
CommandHelp: commandHelp,
|
||||||
|
MetadataInfo: &fs.MetadataInfo{
|
||||||
|
System: systemMetadataInfo,
|
||||||
|
Help: `Depending on which OS is in use the local backend may return only some
|
||||||
|
of the system metadata. Setting system metadata is supported on all
|
||||||
|
OSes but setting user metadata is only supported on linux, freebsd,
|
||||||
|
netbsd, macOS and Solaris. It is **not** supported on Windows yet
|
||||||
|
([see pkg/attrs#47](https://github.com/pkg/xattr/issues/47)).
|
||||||
|
|
||||||
|
User metadata is stored as extended attributes (which may not be
|
||||||
|
supported by all file systems) under the "user.*" prefix.
|
||||||
|
`,
|
||||||
|
},
|
||||||
Options: []fs.Option{{
|
Options: []fs.Option{{
|
||||||
Name: "nounc",
|
Name: "nounc",
|
||||||
Help: "Disable UNC (long path names) conversion on Windows.",
|
Help: "Disable UNC (long path names) conversion on Windows.",
|
||||||
@ -280,6 +292,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
|||||||
CanHaveEmptyDirectories: true,
|
CanHaveEmptyDirectories: true,
|
||||||
IsLocal: true,
|
IsLocal: true,
|
||||||
SlowHash: true,
|
SlowHash: true,
|
||||||
|
ReadMetadata: true,
|
||||||
|
WriteMetadata: true,
|
||||||
|
UserMetadata: xattrSupported, // can only R/W general purpose metadata if xattrs are supported
|
||||||
}).Fill(ctx, f)
|
}).Fill(ctx, f)
|
||||||
if opt.FollowSymlinks {
|
if opt.FollowSymlinks {
|
||||||
f.lstat = os.Stat
|
f.lstat = os.Stat
|
||||||
@ -938,17 +953,22 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
|
|||||||
return o.modTime
|
return o.modTime
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set the atime and ltime of the object
|
||||||
|
func (o *Object) setTimes(atime, mtime time.Time) (err error) {
|
||||||
|
if o.translatedLink {
|
||||||
|
err = lChtimes(o.path, atime, mtime)
|
||||||
|
} else {
|
||||||
|
err = os.Chtimes(o.path, atime, mtime)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// SetModTime sets the modification time of the local fs object
|
// SetModTime sets the modification time of the local fs object
|
||||||
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||||
if o.fs.opt.NoSetModTime {
|
if o.fs.opt.NoSetModTime {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
var err error
|
err := o.setTimes(modTime, modTime)
|
||||||
if o.translatedLink {
|
|
||||||
err = lChtimes(o.path, modTime, modTime)
|
|
||||||
} else {
|
|
||||||
err = os.Chtimes(o.path, modTime, modTime)
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -1223,6 +1243,16 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fetch and set metadata if --metadata is in use
|
||||||
|
meta, err := fs.GetMetadataOptions(ctx, src, options)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to read metadata from source object: %w", err)
|
||||||
|
}
|
||||||
|
err = o.writeMetadata(meta)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to set metadata: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
// ReRead info now that we have finished
|
// ReRead info now that we have finished
|
||||||
return o.lstat()
|
return o.lstat()
|
||||||
}
|
}
|
||||||
@ -1321,6 +1351,34 @@ func (o *Object) Remove(ctx context.Context) error {
|
|||||||
return remove(o.path)
|
return remove(o.path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Metadata returns metadata for an object
|
||||||
|
//
|
||||||
|
// It should return nil if there is no Metadata
|
||||||
|
func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
|
||||||
|
metadata, err = o.getXattr()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
err = o.readMetadataFromFile(&metadata)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write the metadata on the object
|
||||||
|
func (o *Object) writeMetadata(metadata fs.Metadata) (err error) {
|
||||||
|
err = o.setXattr(metadata)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err = o.writeMetadataToFile(metadata)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
|
func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") {
|
if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") {
|
||||||
@ -1360,4 +1418,5 @@ var (
|
|||||||
_ fs.Commander = &Fs{}
|
_ fs.Commander = &Fs{}
|
||||||
_ fs.OpenWriterAter = &Fs{}
|
_ fs.OpenWriterAter = &Fs{}
|
||||||
_ fs.Object = &Object{}
|
_ fs.Object = &Object{}
|
||||||
|
_ fs.Metadataer = &Object{}
|
||||||
)
|
)
|
||||||
|
@ -3,10 +3,12 @@ package local
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -229,3 +231,138 @@ func TestHashOnDelete(t *testing.T) {
|
|||||||
_, err = o.Hash(ctx, hash.MD5)
|
_, err = o.Hash(ctx, hash.MD5)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMetadata(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
r := fstest.NewRun(t)
|
||||||
|
defer r.Finalise()
|
||||||
|
const filePath = "metafile.txt"
|
||||||
|
when := time.Now()
|
||||||
|
const dayLength = len("2001-01-01")
|
||||||
|
whenRFC := when.Format(time.RFC3339Nano)
|
||||||
|
r.WriteFile(filePath, "metadata file contents", when)
|
||||||
|
f := r.Flocal.(*Fs)
|
||||||
|
|
||||||
|
// Get the object
|
||||||
|
obj, err := f.NewObject(ctx, filePath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
o := obj.(*Object)
|
||||||
|
|
||||||
|
features := f.Features()
|
||||||
|
|
||||||
|
var hasXID, hasAtime, hasBtime bool
|
||||||
|
switch runtime.GOOS {
|
||||||
|
case "darwin", "freebsd", "netbsd", "linux":
|
||||||
|
hasXID, hasAtime, hasBtime = true, true, true
|
||||||
|
case "openbsd", "solaris":
|
||||||
|
hasXID, hasAtime = true, true
|
||||||
|
case "windows":
|
||||||
|
hasAtime, hasBtime = true, true
|
||||||
|
case "plan9", "js":
|
||||||
|
// nada
|
||||||
|
default:
|
||||||
|
t.Errorf("No test cases for OS %q", runtime.GOOS)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.True(t, features.ReadMetadata)
|
||||||
|
assert.True(t, features.WriteMetadata)
|
||||||
|
assert.Equal(t, xattrSupported, features.UserMetadata)
|
||||||
|
|
||||||
|
t.Run("Xattr", func(t *testing.T) {
|
||||||
|
if !xattrSupported {
|
||||||
|
t.Skip()
|
||||||
|
}
|
||||||
|
m, err := o.getXattr()
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Nil(t, m)
|
||||||
|
|
||||||
|
inM := fs.Metadata{
|
||||||
|
"potato": "chips",
|
||||||
|
"cabbage": "soup",
|
||||||
|
}
|
||||||
|
err = o.setXattr(inM)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
m, err = o.getXattr()
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotNil(t, m)
|
||||||
|
assert.Equal(t, inM, m)
|
||||||
|
})
|
||||||
|
|
||||||
|
checkTime := func(m fs.Metadata, key string, when time.Time) {
|
||||||
|
mt, ok := o.parseMetadataTime(m, key)
|
||||||
|
assert.True(t, ok)
|
||||||
|
dt := mt.Sub(when)
|
||||||
|
precision := time.Second
|
||||||
|
assert.True(t, dt >= -precision && dt <= precision, fmt.Sprintf("%s: dt %v outside +/- precision %v", key, dt, precision))
|
||||||
|
}
|
||||||
|
|
||||||
|
checkInt := func(m fs.Metadata, key string, base int) int {
|
||||||
|
value, ok := o.parseMetadataInt(m, key, base)
|
||||||
|
assert.True(t, ok)
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
t.Run("Read", func(t *testing.T) {
|
||||||
|
m, err := o.Metadata(ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotNil(t, m)
|
||||||
|
|
||||||
|
// All OSes have these
|
||||||
|
checkInt(m, "mode", 8)
|
||||||
|
checkTime(m, "mtime", when)
|
||||||
|
|
||||||
|
assert.Equal(t, len(whenRFC), len(m["mtime"]))
|
||||||
|
assert.Equal(t, whenRFC[:dayLength], m["mtime"][:dayLength])
|
||||||
|
|
||||||
|
if hasAtime {
|
||||||
|
checkTime(m, "atime", when)
|
||||||
|
}
|
||||||
|
if hasBtime {
|
||||||
|
checkTime(m, "btime", when)
|
||||||
|
}
|
||||||
|
if hasXID {
|
||||||
|
checkInt(m, "uid", 10)
|
||||||
|
checkInt(m, "gid", 10)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("Write", func(t *testing.T) {
|
||||||
|
newAtimeString := "2011-12-13T14:15:16.999999999Z"
|
||||||
|
newAtime := fstest.Time(newAtimeString)
|
||||||
|
newMtimeString := "2011-12-12T14:15:16.999999999Z"
|
||||||
|
newMtime := fstest.Time(newMtimeString)
|
||||||
|
newBtimeString := "2011-12-11T14:15:16.999999999Z"
|
||||||
|
newBtime := fstest.Time(newBtimeString)
|
||||||
|
newM := fs.Metadata{
|
||||||
|
"mtime": newMtimeString,
|
||||||
|
"atime": newAtimeString,
|
||||||
|
"btime": newBtimeString,
|
||||||
|
// Can't test uid, gid without being root
|
||||||
|
"mode": "0767",
|
||||||
|
"potato": "wedges",
|
||||||
|
}
|
||||||
|
err := o.writeMetadata(newM)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
m, err := o.Metadata(ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.NotNil(t, m)
|
||||||
|
|
||||||
|
mode := checkInt(m, "mode", 8)
|
||||||
|
if runtime.GOOS != "windows" {
|
||||||
|
assert.Equal(t, 0767, mode&0777, fmt.Sprintf("mode wrong - expecting 0767 got 0%o", mode&0777))
|
||||||
|
}
|
||||||
|
|
||||||
|
checkTime(m, "mtime", newMtime)
|
||||||
|
if hasAtime {
|
||||||
|
checkTime(m, "atime", newAtime)
|
||||||
|
}
|
||||||
|
if haveSetBTime {
|
||||||
|
checkTime(m, "btime", newBtime)
|
||||||
|
}
|
||||||
|
if xattrSupported {
|
||||||
|
assert.Equal(t, "wedges", m["potato"])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
}
|
||||||
|
138
backend/local/metadata.go
Normal file
138
backend/local/metadata.go
Normal file
@ -0,0 +1,138 @@
|
|||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rclone/rclone/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
const metadataTimeFormat = time.RFC3339Nano
|
||||||
|
|
||||||
|
// system metadata keys which this backend owns
|
||||||
|
//
|
||||||
|
// not all values supported on all OSes
|
||||||
|
var systemMetadataInfo = map[string]fs.MetadataHelp{
|
||||||
|
"mode": {
|
||||||
|
Help: "File type and mode",
|
||||||
|
Type: "octal, unix style",
|
||||||
|
Example: "0100664",
|
||||||
|
},
|
||||||
|
"uid": {
|
||||||
|
Help: "User ID of owner",
|
||||||
|
Type: "decimal number",
|
||||||
|
Example: "500",
|
||||||
|
},
|
||||||
|
"gid": {
|
||||||
|
Help: "Group ID of owner",
|
||||||
|
Type: "decimal number",
|
||||||
|
Example: "500",
|
||||||
|
},
|
||||||
|
"rdev": {
|
||||||
|
Help: "Device ID (if special file)",
|
||||||
|
Type: "hexadecimal",
|
||||||
|
Example: "1abc",
|
||||||
|
},
|
||||||
|
"atime": {
|
||||||
|
Help: "Time of last access",
|
||||||
|
Type: "RFC 3339",
|
||||||
|
Example: "2006-01-02T15:04:05.999999999Z07:00",
|
||||||
|
},
|
||||||
|
"mtime": {
|
||||||
|
Help: "Time of last modification",
|
||||||
|
Type: "RFC 3339",
|
||||||
|
Example: "2006-01-02T15:04:05.999999999Z07:00",
|
||||||
|
},
|
||||||
|
"btime": {
|
||||||
|
Help: "Time of file birth (creation)",
|
||||||
|
Type: "RFC 3339",
|
||||||
|
Example: "2006-01-02T15:04:05.999999999Z07:00",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// parse a time string from metadata with key
|
||||||
|
func (o *Object) parseMetadataTime(m fs.Metadata, key string) (t time.Time, ok bool) {
|
||||||
|
value, ok := m[key]
|
||||||
|
if ok {
|
||||||
|
var err error
|
||||||
|
t, err = time.Parse(metadataTimeFormat, value)
|
||||||
|
if err != nil {
|
||||||
|
fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err)
|
||||||
|
ok = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return t, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// parse am int from metadata with key and base
|
||||||
|
func (o *Object) parseMetadataInt(m fs.Metadata, key string, base int) (result int, ok bool) {
|
||||||
|
value, ok := m[key]
|
||||||
|
if ok {
|
||||||
|
var err error
|
||||||
|
result64, err := strconv.ParseInt(value, base, 64)
|
||||||
|
if err != nil {
|
||||||
|
fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err)
|
||||||
|
ok = false
|
||||||
|
}
|
||||||
|
result = int(result64)
|
||||||
|
}
|
||||||
|
return result, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write the metadata into the file
|
||||||
|
//
|
||||||
|
// It isn't possible to set the ctime and btime under Unix
|
||||||
|
func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
|
||||||
|
var err error
|
||||||
|
atime, atimeOK := o.parseMetadataTime(m, "atime")
|
||||||
|
mtime, mtimeOK := o.parseMetadataTime(m, "mtime")
|
||||||
|
btime, btimeOK := o.parseMetadataTime(m, "btime")
|
||||||
|
if atimeOK || mtimeOK {
|
||||||
|
if atimeOK && !mtimeOK {
|
||||||
|
mtime = atime
|
||||||
|
}
|
||||||
|
if !atimeOK && mtimeOK {
|
||||||
|
atime = mtime
|
||||||
|
}
|
||||||
|
err = o.setTimes(atime, mtime)
|
||||||
|
if err != nil {
|
||||||
|
outErr = fmt.Errorf("failed to set times: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if haveSetBTime {
|
||||||
|
if btimeOK {
|
||||||
|
err = setBTime(o.path, btime)
|
||||||
|
if err != nil {
|
||||||
|
outErr = fmt.Errorf("failed to set birth (creation) time: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
uid, hasUID := o.parseMetadataInt(m, "uid", 10)
|
||||||
|
gid, hasGID := o.parseMetadataInt(m, "gid", 10)
|
||||||
|
if hasUID {
|
||||||
|
// FIXME should read UID and GID of current user and only attempt to set it if different
|
||||||
|
if !hasGID {
|
||||||
|
gid = uid
|
||||||
|
}
|
||||||
|
if runtime.GOOS == "windows" || runtime.GOOS == "plan9" {
|
||||||
|
fs.Debugf(o, "Ignoring request to set ownership %o.%o on this OS", gid, uid)
|
||||||
|
} else {
|
||||||
|
err = os.Chown(o.path, uid, gid)
|
||||||
|
if err != nil {
|
||||||
|
outErr = fmt.Errorf("failed to change ownership: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mode, hasMode := o.parseMetadataInt(m, "mode", 8)
|
||||||
|
if hasMode {
|
||||||
|
err = os.Chmod(o.path, os.FileMode(mode))
|
||||||
|
if err != nil {
|
||||||
|
outErr = fmt.Errorf("failed to change permissions: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// FIXME not parsing rdev yet
|
||||||
|
return outErr
|
||||||
|
}
|
38
backend/local/metadata_bsd.go
Normal file
38
backend/local/metadata_bsd.go
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
//go:build darwin || freebsd || netbsd
|
||||||
|
// +build darwin freebsd netbsd
|
||||||
|
|
||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rclone/rclone/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Read the metadata from the file into metadata where possible
|
||||||
|
func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
|
||||||
|
info, err := o.fs.lstat(o.path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||||
|
if !ok {
|
||||||
|
fs.Debugf(o, "didn't return Stat_t as expected")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
m.Set("mode", fmt.Sprintf("%0o", stat.Mode))
|
||||||
|
m.Set("uid", fmt.Sprintf("%d", stat.Uid))
|
||||||
|
m.Set("gid", fmt.Sprintf("%d", stat.Gid))
|
||||||
|
if stat.Rdev != 0 {
|
||||||
|
m.Set("rdev", fmt.Sprintf("%x", stat.Rdev))
|
||||||
|
}
|
||||||
|
setTime := func(key string, t syscall.Timespec) {
|
||||||
|
m.Set(key, time.Unix(t.Unix()).Format(metadataTimeFormat))
|
||||||
|
}
|
||||||
|
setTime("atime", stat.Atimespec)
|
||||||
|
setTime("mtime", stat.Mtimespec)
|
||||||
|
setTime("btime", stat.Birthtimespec)
|
||||||
|
return nil
|
||||||
|
}
|
47
backend/local/metadata_linux.go
Normal file
47
backend/local/metadata_linux.go
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
//go:build linux
|
||||||
|
// +build linux
|
||||||
|
|
||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rclone/rclone/fs"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Read the metadata from the file into metadata where possible
|
||||||
|
func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
|
||||||
|
flags := unix.AT_SYMLINK_NOFOLLOW
|
||||||
|
if o.fs.opt.FollowSymlinks {
|
||||||
|
flags = 0
|
||||||
|
}
|
||||||
|
var stat unix.Statx_t
|
||||||
|
err = unix.Statx(unix.AT_FDCWD, o.path, flags, (0 |
|
||||||
|
unix.STATX_TYPE | // Want stx_mode & S_IFMT
|
||||||
|
unix.STATX_MODE | // Want stx_mode & ~S_IFMT
|
||||||
|
unix.STATX_UID | // Want stx_uid
|
||||||
|
unix.STATX_GID | // Want stx_gid
|
||||||
|
unix.STATX_ATIME | // Want stx_atime
|
||||||
|
unix.STATX_MTIME | // Want stx_mtime
|
||||||
|
unix.STATX_CTIME | // Want stx_ctime
|
||||||
|
unix.STATX_BTIME), // Want stx_btime
|
||||||
|
&stat)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
m.Set("mode", fmt.Sprintf("%0o", stat.Mode))
|
||||||
|
m.Set("uid", fmt.Sprintf("%d", stat.Uid))
|
||||||
|
m.Set("gid", fmt.Sprintf("%d", stat.Gid))
|
||||||
|
if stat.Rdev_major != 0 || stat.Rdev_minor != 0 {
|
||||||
|
m.Set("rdev", fmt.Sprintf("%x", uint64(stat.Rdev_major)<<32|uint64(stat.Rdev_minor)))
|
||||||
|
}
|
||||||
|
setTime := func(key string, t unix.StatxTimestamp) {
|
||||||
|
m.Set(key, time.Unix(t.Sec, int64(t.Nsec)).Format(metadataTimeFormat))
|
||||||
|
}
|
||||||
|
setTime("atime", stat.Atime)
|
||||||
|
setTime("mtime", stat.Mtime)
|
||||||
|
setTime("btime", stat.Btime)
|
||||||
|
return nil
|
||||||
|
}
|
21
backend/local/metadata_other.go
Normal file
21
backend/local/metadata_other.go
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
//go:build plan9 || js
|
||||||
|
// +build plan9 js
|
||||||
|
|
||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/rclone/rclone/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Read the metadata from the file into metadata where possible
|
||||||
|
func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
|
||||||
|
info, err := o.fs.lstat(o.path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
m.Set("mode", fmt.Sprintf("%0o", info.Mode()))
|
||||||
|
m.Set("mtime", info.ModTime().Format(metadataTimeFormat))
|
||||||
|
return nil
|
||||||
|
}
|
37
backend/local/metadata_unix.go
Normal file
37
backend/local/metadata_unix.go
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
//go:build openbsd || solaris
|
||||||
|
// +build openbsd solaris
|
||||||
|
|
||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rclone/rclone/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Read the metadata from the file into metadata where possible
|
||||||
|
func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
|
||||||
|
info, err := o.fs.lstat(o.path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||||
|
if !ok {
|
||||||
|
fs.Debugf(o, "didn't return Stat_t as expected")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
m.Set("mode", fmt.Sprintf("%0o", stat.Mode))
|
||||||
|
m.Set("uid", fmt.Sprintf("%d", stat.Uid))
|
||||||
|
m.Set("gid", fmt.Sprintf("%d", stat.Gid))
|
||||||
|
if stat.Rdev != 0 {
|
||||||
|
m.Set("rdev", fmt.Sprintf("%x", stat.Rdev))
|
||||||
|
}
|
||||||
|
setTime := func(key string, t syscall.Timespec) {
|
||||||
|
m.Set(key, time.Unix(t.Unix()).Format(metadataTimeFormat))
|
||||||
|
}
|
||||||
|
setTime("atime", stat.Atim)
|
||||||
|
setTime("mtime", stat.Mtim)
|
||||||
|
return nil
|
||||||
|
}
|
34
backend/local/metadata_windows.go
Normal file
34
backend/local/metadata_windows.go
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
//go:build windows
|
||||||
|
// +build windows
|
||||||
|
|
||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rclone/rclone/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Read the metadata from the file into metadata where possible
|
||||||
|
func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
|
||||||
|
info, err := o.fs.lstat(o.path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
stat, ok := info.Sys().(*syscall.Win32FileAttributeData)
|
||||||
|
if !ok {
|
||||||
|
fs.Debugf(o, "didn't return Win32FileAttributeData as expected")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// FIXME do something with stat.FileAttributes ?
|
||||||
|
m.Set("mode", fmt.Sprintf("%0o", info.Mode()))
|
||||||
|
setTime := func(key string, t syscall.Filetime) {
|
||||||
|
m.Set(key, time.Unix(0, t.Nanoseconds()).Format(metadataTimeFormat))
|
||||||
|
}
|
||||||
|
setTime("atime", stat.LastAccessTime)
|
||||||
|
setTime("mtime", stat.LastWriteTime)
|
||||||
|
setTime("btime", stat.CreationTime)
|
||||||
|
return nil
|
||||||
|
}
|
16
backend/local/setbtime.go
Normal file
16
backend/local/setbtime.go
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
//go:build !windows
|
||||||
|
// +build !windows
|
||||||
|
|
||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const haveSetBTime = false
|
||||||
|
|
||||||
|
// setBTime changes the the birth time of the file passed in
|
||||||
|
func setBTime(name string, btime time.Time) error {
|
||||||
|
// Does nothing
|
||||||
|
return nil
|
||||||
|
}
|
28
backend/local/setbtime_windows.go
Normal file
28
backend/local/setbtime_windows.go
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
//go:build windows
|
||||||
|
// +build windows
|
||||||
|
|
||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
const haveSetBTime = true
|
||||||
|
|
||||||
|
// setBTime sets the the birth time of the file passed in
|
||||||
|
func setBTime(name string, btime time.Time) (err error) {
|
||||||
|
h, err := syscall.Open(name, os.O_RDWR, 0755)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
closeErr := syscall.Close(h)
|
||||||
|
if err == nil {
|
||||||
|
err = closeErr
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
bFileTime := syscall.NsecToFiletime(btime.UnixNano())
|
||||||
|
return syscall.SetFileTime(h, &bFileTime, nil, nil)
|
||||||
|
}
|
87
backend/local/xattr.go
Normal file
87
backend/local/xattr.go
Normal file
@ -0,0 +1,87 @@
|
|||||||
|
//go:build !openbsd && !plan9
|
||||||
|
// +build !openbsd,!plan9
|
||||||
|
|
||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/pkg/xattr"
|
||||||
|
"github.com/rclone/rclone/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
xattrPrefix = "user." // FIXME is this correct for all unixes?
|
||||||
|
xattrSupported = xattr.XATTR_SUPPORTED
|
||||||
|
)
|
||||||
|
|
||||||
|
// getXattr returns the extended attributes for an object
|
||||||
|
//
|
||||||
|
// It doesn't return any attributes owned by this backend in
|
||||||
|
// metadataKeys
|
||||||
|
func (o *Object) getXattr() (metadata fs.Metadata, err error) {
|
||||||
|
if !xattrSupported {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var list []string
|
||||||
|
if o.fs.opt.FollowSymlinks {
|
||||||
|
list, err = xattr.List(o.path)
|
||||||
|
} else {
|
||||||
|
list, err = xattr.LList(o.path)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read xattr: %w", err)
|
||||||
|
}
|
||||||
|
if len(list) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
metadata = make(fs.Metadata, len(list))
|
||||||
|
for _, k := range list {
|
||||||
|
var v []byte
|
||||||
|
if o.fs.opt.FollowSymlinks {
|
||||||
|
v, err = xattr.Get(o.path, k)
|
||||||
|
} else {
|
||||||
|
v, err = xattr.LGet(o.path, k)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read xattr key %q: %w", k, err)
|
||||||
|
}
|
||||||
|
k = strings.ToLower(k)
|
||||||
|
if !strings.HasPrefix(k, xattrPrefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
k = k[len(xattrPrefix):]
|
||||||
|
if _, found := systemMetadataInfo[k]; found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
metadata[k] = string(v)
|
||||||
|
}
|
||||||
|
return metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// setXattr sets the metadata on the file Xattrs
|
||||||
|
//
|
||||||
|
// It doesn't set any attributes owned by this backend in metadataKeys
|
||||||
|
func (o *Object) setXattr(metadata fs.Metadata) (err error) {
|
||||||
|
if !xattrSupported {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for k, value := range metadata {
|
||||||
|
k = strings.ToLower(k)
|
||||||
|
if _, found := systemMetadataInfo[k]; found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
k = xattrPrefix + k
|
||||||
|
v := []byte(value)
|
||||||
|
if o.fs.opt.FollowSymlinks {
|
||||||
|
err = xattr.Set(o.path, k, v)
|
||||||
|
} else {
|
||||||
|
err = xattr.LSet(o.path, k, v)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to set xattr key %q: %w", k, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
21
backend/local/xattr_unsupported.go
Normal file
21
backend/local/xattr_unsupported.go
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
//go:build openbsd || plan9
|
||||||
|
// +build openbsd plan9
|
||||||
|
|
||||||
|
// The pkg/xattr module doesn't compile for openbsd or plan9
|
||||||
|
package local
|
||||||
|
|
||||||
|
import "github.com/rclone/rclone/fs"
|
||||||
|
|
||||||
|
const (
|
||||||
|
xattrSupported = false
|
||||||
|
)
|
||||||
|
|
||||||
|
// getXattr returns the extended attributes for an object
|
||||||
|
func (o *Object) getXattr() (metadata fs.Metadata, err error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// setXattr sets the metadata on the file Xattrs
|
||||||
|
func (o *Object) setXattr(metadata fs.Metadata) (err error) {
|
||||||
|
return nil
|
||||||
|
}
|
@ -563,6 +563,32 @@ Properties:
|
|||||||
- Type: MultiEncoder
|
- Type: MultiEncoder
|
||||||
- Default: Slash,Dot
|
- Default: Slash,Dot
|
||||||
|
|
||||||
|
### Metadata
|
||||||
|
|
||||||
|
Depending on which OS is in use the local backend may return only some
|
||||||
|
of the system metadata. Setting system metadata is supported on all
|
||||||
|
OSes but setting user metadata is only supported on linux, freebsd,
|
||||||
|
netbsd, macOS and Solaris. It is **not** supported on Windows yet
|
||||||
|
([see pkg/attrs#47](https://github.com/pkg/xattr/issues/47)).
|
||||||
|
|
||||||
|
User metadata is stored as extended attributes (which may not be
|
||||||
|
supported by all file systems) under the "user.*" prefix.
|
||||||
|
|
||||||
|
Here are the possible system metadata items for the local backend.
|
||||||
|
|
||||||
|
| Name | Help | Type | Example | Read Only |
|
||||||
|
|------|------|------|---------|-----------|
|
||||||
|
| atime | Time of last access | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N |
|
||||||
|
| btime | Time of file birth (creation) | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N |
|
||||||
|
| gid | Group ID of owner | decimal number | 500 | N |
|
||||||
|
| mode | File type and mode | octal, unix style | 0100664 | N |
|
||||||
|
| mtime | Time of last modification | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N |
|
||||||
|
| rdev | Device ID (if special file) | hexadecimal | 1abc | N |
|
||||||
|
| uid | User ID of owner | decimal number | 500 | N |
|
||||||
|
|
||||||
|
|
||||||
|
See the [metadata](/docs/#metadata) docs for more info.
|
||||||
|
|
||||||
## Backend commands
|
## Backend commands
|
||||||
|
|
||||||
Here are the commands specific to the local backend.
|
Here are the commands specific to the local backend.
|
||||||
|
1
go.mod
1
go.mod
@ -76,6 +76,7 @@ require (
|
|||||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||||
github.com/google/go-querystring v1.1.0 // indirect
|
github.com/google/go-querystring v1.1.0 // indirect
|
||||||
github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af
|
github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af
|
||||||
|
github.com/pkg/xattr v0.4.7 // indirect
|
||||||
golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd
|
golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd
|
||||||
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
|
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
|
||||||
)
|
)
|
||||||
|
3
go.sum
3
go.sum
@ -478,6 +478,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
|||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pkg/sftp v1.13.5-0.20211228200725-31aac3e1878d h1:7cHNeARnMq3icpbMdvyUELykWM4zOj5NRhH2Y3sfgBc=
|
github.com/pkg/sftp v1.13.5-0.20211228200725-31aac3e1878d h1:7cHNeARnMq3icpbMdvyUELykWM4zOj5NRhH2Y3sfgBc=
|
||||||
github.com/pkg/sftp v1.13.5-0.20211228200725-31aac3e1878d/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
|
github.com/pkg/sftp v1.13.5-0.20211228200725-31aac3e1878d/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
|
||||||
|
github.com/pkg/xattr v0.4.7 h1:XoA3KzmFvyPlH4RwX5eMcgtzcaGBaSvgt3IoFQfbrmQ=
|
||||||
|
github.com/pkg/xattr v0.4.7/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
|
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
|
||||||
@ -882,6 +884,7 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220318055525-2edf467146b5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220318055525-2edf467146b5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
|
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
|
||||||
|
Loading…
Reference in New Issue
Block a user