mirror of
https://github.com/rclone/rclone.git
synced 2024-11-23 00:43:49 +01:00
2b7957cc74
This stops the cache cleaner running unnecessarily and saves resources. This also helps with issue #2227 which was caused by a second mount deleting objects in the first mounts cache.
554 lines
13 KiB
Go
554 lines
13 KiB
Go
package vfs
|
|
|
|
import (
|
|
"os"
|
|
"path"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/ncw/rclone/fs"
|
|
"github.com/ncw/rclone/fs/log"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
// File represents a file
|
|
type File struct {
|
|
inode uint64 // inode number
|
|
size int64 // size of file - read and written with atomic int64 - must be 64 bit aligned
|
|
d *Dir // parent directory - read only
|
|
|
|
mu sync.Mutex // protects the following
|
|
o fs.Object // NB o may be nil if file is being written
|
|
leaf string // leaf name of the object
|
|
rwOpenCount int // number of open files on this handle
|
|
writers []Handle // writers for this file
|
|
nwriters int32 // len(writers) which is read/updated with atomic
|
|
readWriters int // how many RWFileHandle are open for writing
|
|
readWriterClosing bool // is a RWFileHandle currently cosing?
|
|
modified bool // has the cache file be modified by a RWFileHandle?
|
|
pendingModTime time.Time // will be applied once o becomes available, i.e. after file was written
|
|
|
|
muRW sync.Mutex // synchonize RWFileHandle.openPending(), RWFileHandle.close() and File.Remove
|
|
}
|
|
|
|
// newFile creates a new File
|
|
func newFile(d *Dir, o fs.Object, leaf string) *File {
|
|
return &File{
|
|
d: d,
|
|
o: o,
|
|
leaf: leaf,
|
|
inode: newInode(),
|
|
}
|
|
}
|
|
|
|
// String converts it to printable
|
|
func (f *File) String() string {
|
|
if f == nil {
|
|
return "<nil *File>"
|
|
}
|
|
return f.Path()
|
|
}
|
|
|
|
// IsFile returns true for File - satisfies Node interface
|
|
func (f *File) IsFile() bool {
|
|
return true
|
|
}
|
|
|
|
// IsDir returns false for File - satisfies Node interface
|
|
func (f *File) IsDir() bool {
|
|
return false
|
|
}
|
|
|
|
// Mode bits of the file or directory - satisfies Node interface
|
|
func (f *File) Mode() (mode os.FileMode) {
|
|
return f.d.vfs.Opt.FilePerms
|
|
}
|
|
|
|
// Name (base) of the directory - satisfies Node interface
|
|
func (f *File) Name() (name string) {
|
|
return f.leaf
|
|
}
|
|
|
|
// Path returns the full path of the file
|
|
func (f *File) Path() string {
|
|
return path.Join(f.d.path, f.leaf)
|
|
}
|
|
|
|
// Sys returns underlying data source (can be nil) - satisfies Node interface
|
|
func (f *File) Sys() interface{} {
|
|
return nil
|
|
}
|
|
|
|
// Inode returns the inode number - satisfies Node interface
|
|
func (f *File) Inode() uint64 {
|
|
return f.inode
|
|
}
|
|
|
|
// Node returns the Node assocuated with this - satisfies Noder interface
|
|
func (f *File) Node() Node {
|
|
return f
|
|
}
|
|
|
|
// rename should be called to update the internals after a rename
|
|
func (f *File) rename(d *Dir, o fs.Object) {
|
|
f.mu.Lock()
|
|
f.o = o
|
|
f.d = d
|
|
f.leaf = path.Base(o.Remote())
|
|
f.mu.Unlock()
|
|
}
|
|
|
|
// addWriter adds a write handle to the file
|
|
func (f *File) addWriter(h Handle) {
|
|
f.mu.Lock()
|
|
f.writers = append(f.writers, h)
|
|
atomic.AddInt32(&f.nwriters, 1)
|
|
if _, ok := h.(*RWFileHandle); ok {
|
|
f.readWriters++
|
|
}
|
|
f.mu.Unlock()
|
|
}
|
|
|
|
// delWriter removes a write handle from the file
|
|
func (f *File) delWriter(h Handle, modifiedCacheFile bool) (lastWriterAndModified bool) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
var found = -1
|
|
for i := range f.writers {
|
|
if f.writers[i] == h {
|
|
found = i
|
|
break
|
|
}
|
|
}
|
|
if found >= 0 {
|
|
f.writers = append(f.writers[:found], f.writers[found+1:]...)
|
|
atomic.AddInt32(&f.nwriters, -1)
|
|
} else {
|
|
fs.Debugf(f.o, "File.delWriter couldn't find handle")
|
|
}
|
|
if _, ok := h.(*RWFileHandle); ok {
|
|
f.readWriters--
|
|
}
|
|
f.readWriterClosing = true
|
|
if modifiedCacheFile {
|
|
f.modified = true
|
|
}
|
|
lastWriterAndModified = len(f.writers) == 0 && f.modified
|
|
if lastWriterAndModified {
|
|
f.modified = false
|
|
}
|
|
return
|
|
}
|
|
|
|
// addRWOpen should be called by ReadWriteHandle when they have
|
|
// actually opened the file for read or write.
|
|
func (f *File) addRWOpen() {
|
|
f.mu.Lock()
|
|
f.rwOpenCount++
|
|
f.mu.Unlock()
|
|
}
|
|
|
|
// delRWOpen should be called by ReadWriteHandle when they have closed
|
|
// an actually opene file for read or write.
|
|
func (f *File) delRWOpen() {
|
|
f.mu.Lock()
|
|
f.rwOpenCount--
|
|
f.mu.Unlock()
|
|
}
|
|
|
|
// rwOpens returns how many active open ReadWriteHandles there are.
|
|
// Note that file handles which are in pending open state aren't
|
|
// counted.
|
|
func (f *File) rwOpens() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.rwOpenCount
|
|
}
|
|
|
|
// finishWriterClose resets the readWriterClosing flag
|
|
func (f *File) finishWriterClose() {
|
|
f.mu.Lock()
|
|
f.readWriterClosing = false
|
|
f.mu.Unlock()
|
|
}
|
|
|
|
// activeWriters returns the number of writers on the file
|
|
//
|
|
// Note that we don't take the mutex here. If we do then we can get a
|
|
// deadlock.
|
|
func (f *File) activeWriters() int {
|
|
return int(atomic.LoadInt32(&f.nwriters))
|
|
}
|
|
|
|
// ModTime returns the modified time of the file
|
|
//
|
|
// if NoModTime is set then it returns the mod time of the directory
|
|
func (f *File) ModTime() (modTime time.Time) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
|
|
if !f.d.vfs.Opt.NoModTime {
|
|
// if o is nil it isn't valid yet or there are writers, so return the size so far
|
|
if f.o == nil || len(f.writers) != 0 || f.readWriterClosing {
|
|
if !f.pendingModTime.IsZero() {
|
|
return f.pendingModTime
|
|
}
|
|
} else {
|
|
return f.o.ModTime()
|
|
}
|
|
}
|
|
|
|
return f.d.modTime
|
|
}
|
|
|
|
// nonNegative returns 0 if i is -ve, i otherwise
|
|
func nonNegative(i int64) int64 {
|
|
if i >= 0 {
|
|
return i
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// Size of the file
|
|
func (f *File) Size() int64 {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
|
|
// if o is nil it isn't valid yet or there are writers, so return the size so far
|
|
if f.o == nil || len(f.writers) != 0 || f.readWriterClosing {
|
|
return atomic.LoadInt64(&f.size)
|
|
}
|
|
return nonNegative(f.o.Size())
|
|
}
|
|
|
|
// SetModTime sets the modtime for the file
|
|
func (f *File) SetModTime(modTime time.Time) error {
|
|
if f.d.vfs.Opt.ReadOnly {
|
|
return EROFS
|
|
}
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
|
|
f.pendingModTime = modTime
|
|
|
|
// Only update the ModTime when there are no writers, setObject will do it
|
|
if f.o != nil && len(f.writers) == 0 && !f.readWriterClosing {
|
|
return f.applyPendingModTime()
|
|
}
|
|
|
|
// queue up for later, hoping f.o becomes available
|
|
return nil
|
|
}
|
|
|
|
// call with the mutex held
|
|
func (f *File) applyPendingModTime() error {
|
|
defer func() { f.pendingModTime = time.Time{} }()
|
|
|
|
if f.pendingModTime.IsZero() {
|
|
return nil
|
|
}
|
|
|
|
if f.o == nil {
|
|
return errors.New("Cannot apply ModTime, file object is not available")
|
|
}
|
|
|
|
err := f.o.SetModTime(f.pendingModTime)
|
|
switch err {
|
|
case nil:
|
|
fs.Debugf(f.o, "File.applyPendingModTime OK")
|
|
case fs.ErrorCantSetModTime, fs.ErrorCantSetModTimeWithoutDelete:
|
|
// do nothing, in order to not break "touch somefile" if it exists already
|
|
default:
|
|
fs.Errorf(f, "File.applyPendingModTime error: %v", err)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Update the size while writing
|
|
func (f *File) setSize(n int64) {
|
|
atomic.StoreInt64(&f.size, n)
|
|
}
|
|
|
|
// Update the object when written and add it to the directory
|
|
func (f *File) setObject(o fs.Object) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.o = o
|
|
_ = f.applyPendingModTime()
|
|
f.d.addObject(f)
|
|
}
|
|
|
|
// Update the object but don't update the directory cache - for use by
|
|
// the directory cache
|
|
func (f *File) setObjectNoUpdate(o fs.Object) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.o = o
|
|
}
|
|
|
|
// Get the current fs.Object - may be nil
|
|
func (f *File) getObject() fs.Object {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.o
|
|
}
|
|
|
|
// exists returns whether the file exists already
|
|
func (f *File) exists() bool {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.o != nil
|
|
}
|
|
|
|
// Wait for f.o to become non nil for a short time returning it or an
|
|
// error. Use when opening a read handle.
|
|
//
|
|
// Call without the mutex held
|
|
func (f *File) waitForValidObject() (o fs.Object, err error) {
|
|
for i := 0; i < 50; i++ {
|
|
f.mu.Lock()
|
|
o = f.o
|
|
nwriters := len(f.writers)
|
|
wclosing := f.readWriterClosing
|
|
f.mu.Unlock()
|
|
if o != nil {
|
|
return o, nil
|
|
}
|
|
if nwriters == 0 && !wclosing {
|
|
return nil, errors.New("can't open file - writer failed")
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
return nil, ENOENT
|
|
}
|
|
|
|
// openRead open the file for read
|
|
func (f *File) openRead() (fh *ReadFileHandle, err error) {
|
|
// if o is nil it isn't valid yet
|
|
_, err = f.waitForValidObject()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// fs.Debugf(o, "File.openRead")
|
|
|
|
fh, err = newReadFileHandle(f)
|
|
if err != nil {
|
|
fs.Errorf(f, "File.openRead failed: %v", err)
|
|
return nil, err
|
|
}
|
|
return fh, nil
|
|
}
|
|
|
|
// openWrite open the file for write
|
|
func (f *File) openWrite(flags int) (fh *WriteFileHandle, err error) {
|
|
if f.d.vfs.Opt.ReadOnly {
|
|
return nil, EROFS
|
|
}
|
|
// fs.Debugf(o, "File.openWrite")
|
|
|
|
fh, err = newWriteFileHandle(f.d, f, f.Path(), flags)
|
|
if err != nil {
|
|
fs.Errorf(f, "File.openWrite failed: %v", err)
|
|
return nil, err
|
|
}
|
|
return fh, nil
|
|
}
|
|
|
|
// openRW open the file for read and write using a temporay file
|
|
//
|
|
// It uses the open flags passed in.
|
|
func (f *File) openRW(flags int) (fh *RWFileHandle, err error) {
|
|
// FIXME chunked
|
|
if flags&accessModeMask != os.O_RDONLY && f.d.vfs.Opt.ReadOnly {
|
|
return nil, EROFS
|
|
}
|
|
// fs.Debugf(o, "File.openRW")
|
|
|
|
fh, err = newRWFileHandle(f.d, f, f.Path(), flags)
|
|
if err != nil {
|
|
fs.Errorf(f, "File.openRW failed: %v", err)
|
|
return nil, err
|
|
}
|
|
return fh, nil
|
|
}
|
|
|
|
// Sync the file
|
|
//
|
|
// Note that we don't do anything except return OK
|
|
func (f *File) Sync() error {
|
|
return nil
|
|
}
|
|
|
|
// Remove the file
|
|
func (f *File) Remove() error {
|
|
f.muRW.Lock()
|
|
defer f.muRW.Unlock()
|
|
if f.d.vfs.Opt.ReadOnly {
|
|
return EROFS
|
|
}
|
|
if f.o != nil {
|
|
err := f.o.Remove()
|
|
if err != nil {
|
|
fs.Errorf(f, "File.Remove file error: %v", err)
|
|
return err
|
|
}
|
|
}
|
|
// Remove the item from the directory listing
|
|
f.d.delObject(f.Name())
|
|
// Remove the object from the cache
|
|
if f.d.vfs.Opt.CacheMode >= CacheModeMinimal {
|
|
f.d.vfs.cache.remove(f.Path())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RemoveAll the file - same as remove for files
|
|
func (f *File) RemoveAll() error {
|
|
return f.Remove()
|
|
}
|
|
|
|
// DirEntry returns the underlying fs.DirEntry - may be nil
|
|
func (f *File) DirEntry() (entry fs.DirEntry) {
|
|
return f.o
|
|
}
|
|
|
|
// Dir returns the directory this file is in
|
|
func (f *File) Dir() *Dir {
|
|
return f.d
|
|
}
|
|
|
|
// VFS returns the instance of the VFS
|
|
func (f *File) VFS() *VFS {
|
|
return f.d.vfs
|
|
}
|
|
|
|
// Open a file according to the flags provided
|
|
//
|
|
// O_RDONLY open the file read-only.
|
|
// O_WRONLY open the file write-only.
|
|
// O_RDWR open the file read-write.
|
|
//
|
|
// O_APPEND append data to the file when writing.
|
|
// O_CREATE create a new file if none exists.
|
|
// O_EXCL used with O_CREATE, file must not exist
|
|
// O_SYNC open for synchronous I/O.
|
|
// O_TRUNC if possible, truncate file when opene
|
|
//
|
|
// We ignore O_SYNC and O_EXCL
|
|
func (f *File) Open(flags int) (fd Handle, err error) {
|
|
defer log.Trace(f, "flags=%s", decodeOpenFlags(flags))("fd=%v, err=%v", &fd, &err)
|
|
var (
|
|
write bool // if set need write support
|
|
read bool // if set need read support
|
|
rdwrMode = flags & accessModeMask
|
|
)
|
|
|
|
// http://pubs.opengroup.org/onlinepubs/7908799/xsh/open.html
|
|
// The result of using O_TRUNC with O_RDONLY is undefined.
|
|
// Linux seems to truncate the file, but we prefer to return EINVAL
|
|
if rdwrMode == os.O_RDONLY && flags&os.O_TRUNC != 0 {
|
|
return nil, EINVAL
|
|
}
|
|
|
|
// Figure out the read/write intents
|
|
switch {
|
|
case rdwrMode == os.O_RDONLY:
|
|
read = true
|
|
case rdwrMode == os.O_WRONLY:
|
|
write = true
|
|
case rdwrMode == os.O_RDWR:
|
|
read = true
|
|
write = true
|
|
default:
|
|
fs.Errorf(f, "Can't figure out how to open with flags: 0x%X", flags)
|
|
return nil, EPERM
|
|
}
|
|
|
|
// If append is set then set read to force openRW
|
|
if flags&os.O_APPEND != 0 {
|
|
read = true
|
|
}
|
|
|
|
// If truncate is set then set write to force openRW
|
|
if flags&os.O_TRUNC != 0 {
|
|
write = true
|
|
}
|
|
|
|
// FIXME discover if file is in cache or not?
|
|
|
|
// Open the correct sort of handle
|
|
CacheMode := f.d.vfs.Opt.CacheMode
|
|
if CacheMode >= CacheModeMinimal && f.d.vfs.cache.opens(f.Path()) > 0 {
|
|
fd, err = f.openRW(flags)
|
|
} else if read && write {
|
|
if CacheMode >= CacheModeMinimal {
|
|
fd, err = f.openRW(flags)
|
|
} else {
|
|
// Open write only and hope the user doesn't
|
|
// want to read. If they do they will get an
|
|
// EPERM plus an Error log.
|
|
fd, err = f.openWrite(flags)
|
|
}
|
|
} else if write {
|
|
if CacheMode >= CacheModeWrites {
|
|
fd, err = f.openRW(flags)
|
|
} else {
|
|
fd, err = f.openWrite(flags)
|
|
}
|
|
} else if read {
|
|
if CacheMode >= CacheModeFull {
|
|
fd, err = f.openRW(flags)
|
|
} else {
|
|
fd, err = f.openRead()
|
|
}
|
|
} else {
|
|
fs.Errorf(f, "Can't figure out how to open with flags: 0x%X", flags)
|
|
return nil, EPERM
|
|
}
|
|
return fd, err
|
|
}
|
|
|
|
// Truncate changes the size of the named file.
|
|
func (f *File) Truncate(size int64) (err error) {
|
|
// make a copy of fh.writers with the lock held then unlock so
|
|
// we can call other file methods.
|
|
f.mu.Lock()
|
|
writers := make([]Handle, len(f.writers))
|
|
copy(writers, f.writers)
|
|
f.mu.Unlock()
|
|
|
|
// FIXME: handle closing writer
|
|
|
|
// If have writers then call truncate for each writer
|
|
if len(writers) != 0 {
|
|
fs.Debugf(f.o, "Truncating %d file handles", len(writers))
|
|
for _, h := range writers {
|
|
truncateErr := h.Truncate(size)
|
|
if truncateErr != nil {
|
|
err = truncateErr
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
fs.Debugf(f.o, "Truncating file")
|
|
|
|
// Otherwise if no writers then truncate the file by opening
|
|
// the file and truncating it.
|
|
flags := os.O_WRONLY
|
|
if size == 0 {
|
|
flags |= os.O_TRUNC
|
|
}
|
|
fh, err := f.Open(flags)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer fs.CheckClose(fh, &err)
|
|
if size != 0 {
|
|
return fh.Truncate(size)
|
|
}
|
|
return nil
|
|
}
|