ftp: add support for precise time #5655

backend/ftp/ftp.go

@@ -98,6 +98,11 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Help:     "Disable using MLSD even if server advertises support.",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "writing_mdtm",
+			Help:     "Use MDTM to set modification time (VsFtpd quirk)",
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -169,6 +174,7 @@ type Options struct {
 	SkipVerifyTLSCert bool                 `config:"no_check_certificate"`
 	DisableEPSV       bool                 `config:"disable_epsv"`
 	DisableMLSD       bool                 `config:"disable_mlsd"`
+	WritingMDTM       bool                 `config:"writing_mdtm"`
 	IdleTimeout       fs.Duration          `config:"idle_timeout"`
 	CloseTimeout      fs.Duration          `config:"close_timeout"`
 	ShutTimeout       fs.Duration          `config:"shut_timeout"`
@@ -192,6 +198,9 @@ type Fs struct {
 	tokens   *pacer.TokenDispenser
 	tlsConf  *tls.Config
 	pacer    *fs.Pacer // pacer for FTP connections
+	fGetTime bool      // true if the ftp library accepts GetTime
+	fSetTime bool      // true if the ftp library accepts SetTime
+	fLstTime bool      // true if the List call returns precise time
 }
 
 // Object describes an FTP file
@@ -206,6 +215,7 @@ type FileInfo struct {
 	Name    string
 	Size    uint64
 	ModTime time.Time
+	precise bool // true if the time is precise
 	IsDir   bool
 }
 
@@ -320,6 +330,9 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	if f.opt.ShutTimeout != 0 && f.opt.ShutTimeout != fs.DurationOff {
 		ftpConfig = append(ftpConfig, ftp.DialWithShutTimeout(time.Duration(f.opt.ShutTimeout)))
 	}
+	if f.opt.WritingMDTM {
+		ftpConfig = append(ftpConfig, ftp.DialWithWritingMDTM(true))
+	}
 	if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
 		ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0}))
 	}
@@ -491,6 +504,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	if err != nil {
 		return nil, errors.Wrap(err, "NewFs")
 	}
+	f.fGetTime = c.IsGetTimeSupported()
+	f.fSetTime = c.IsSetTimeSupported()
+	f.fLstTime = c.IsTimePreciseInList()
+	if !f.fLstTime && f.fGetTime {
+		f.features.SlowModTime = true
+	}
 	f.putFtpConnection(&c, nil)
 	if root != "" {
 		// Check to see if the root actually an existing file
@@ -609,13 +628,12 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err err
 			fs:     f,
 			remote: remote,
 		}
-		info := &FileInfo{
+		o.info = &FileInfo{
 			Name:    remote,
 			Size:    entry.Size,
 			ModTime: entry.Time,
+			precise: f.fLstTime,
 		}
-		o.info = info
-
 		return o, nil
 	}
 	return nil, fs.ErrorObjectNotFound
@@ -710,6 +728,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				Name:    newremote,
 				Size:    object.Size,
 				ModTime: object.Time,
+				precise: f.fLstTime,
 			}
 			o.info = info
 			entries = append(entries, o)
@@ -723,8 +742,18 @@ func (f *Fs) Hashes() hash.Set {
 	return 0
 }
 
-// Precision shows Modified Time not supported
+// Precision shows whether modified time is supported or not depending on the
+// FTP server capabilities, namely whether FTP server:
+// - accepts the MDTM command to get file time (fGetTime)
+//   or supports MLSD returning precise file time in the list (fLstTime)
+// - accepts the MFMT command to set file time (fSetTime)
+//   or non-standard form of the MDTM command (fSetTime, too)
+//   used by VsFtpd for the same purpose (WritingMDTM)
+// See "mdtm_write" in https://security.appspot.com/vsftpd/vsftpd_conf.html
 func (f *Fs) Precision() time.Duration {
+	if (f.fGetTime || f.fLstTime) && f.fSetTime {
+		return time.Second
+	}
 	return fs.ModTimeNotSupported
 }
 
@@ -776,6 +805,7 @@ func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err erro
 				Name:    remote,
 				Size:    file.Size,
 				ModTime: file.Time,
+				precise: f.fLstTime,
 				IsDir:   file.Type == ftp.EntryTypeFolder,
 			}
 			return info, nil
@@ -961,12 +991,41 @@ func (o *Object) Size() int64 {
 
 // ModTime returns the modification time of the object
 func (o *Object) ModTime(ctx context.Context) time.Time {
+	if !o.info.precise && o.fs.fGetTime {
+		c, err := o.fs.getFtpConnection(ctx)
+		if err == nil {
+			path := path.Join(o.fs.root, o.remote)
+			path = o.fs.opt.Enc.FromStandardPath(path)
+			modTime, err := c.GetTime(path)
+			if err == nil && o.info != nil {
+				o.info.ModTime = modTime
+				o.info.precise = true
+			}
+			o.fs.putFtpConnection(&c, err)
+		}
+	}
 	return o.info.ModTime
 }
 
 // SetModTime sets the modification time of the object
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	return nil
+	if !o.fs.fSetTime {
+		fs.Errorf(o.fs, "SetModTime is not supported")
+		return nil
+	}
+	c, err := o.fs.getFtpConnection(ctx)
+	if err != nil {
+		return err
+	}
+	path := path.Join(o.fs.root, o.remote)
+	path = o.fs.opt.Enc.FromStandardPath(path)
+	err = c.SetTime(path, modTime.In(time.UTC))
+	if err == nil && o.info != nil {
+		o.info.ModTime = modTime
+		o.info.precise = true
+	}
+	o.fs.putFtpConnection(&c, err)
+	return err
 }
 
 // Storable returns a boolean as to whether this object is storable
@@ -1108,6 +1167,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return errors.Wrap(err, "update stor")
 	}
 	o.fs.putFtpConnection(&c, nil)
+	if err = o.SetModTime(ctx, src.ModTime(ctx)); err != nil {
+		return errors.Wrap(err, "SetModTime")
+	}
 	o.info, err = o.fs.getInfo(ctx, path)
 	if err != nil {
 		return errors.Wrap(err, "update getinfo")

backend/ftp/ftp_internal_test.go

@@ -90,9 +90,26 @@ func (f *Fs) testUploadTimeout(t *testing.T) {
 	}
 }
 
+// rclone must support precise time with ProFtpd and PureFtpd out of the box.
+// The VsFtpd server does not support the MFMT command to set file time like
+// other servers but by default supports the MDTM command in the non-standard
+// two-argument form for the same purpose.
+// See "mdtm_write" in https://security.appspot.com/vsftpd/vsftpd_conf.html
+func (f *Fs) testTimePrecision(t *testing.T) {
+	name := f.Name()
+	if pos := strings.Index(name, "{"); pos != -1 {
+		name = name[:pos]
+	}
+	switch name {
+	case "TestFTPProftpd", "TestFTPPureftpd", "TestFTPVsftpd":
+		assert.LessOrEqual(t, f.Precision(), time.Second)
+	}
+}
+
 // InternalTest dispatches all internal tests
 func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("UploadTimeout", f.testUploadTimeout)
+	t.Run("TimePrecision", f.testTimePrecision)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

docs/content/ftp.md

@@ -246,6 +246,15 @@ Disable using MLSD even if server advertises support
 - Type:        bool
 - Default:     false
 
+#### --ftp-writing-mdtm
+
+Use MDTM to set modification time (VsFtpd quirk)
+
+- Config:      writing_mdtm
+- Env Var:     RCLONE_FTP_WRITING_MDTM
+- Type:        bool
+- Default:     false
+
 #### --ftp-idle-timeout
 
 Max time before closing idle connections
@@ -298,9 +307,6 @@ Rclone's FTP implementation is not compatible with `active` mode
 as [the library it uses doesn't support it](https://github.com/jlaffaye/ftp/issues/29).
 This will likely never be supported due to security concerns.
 
-Modified times are not supported. Times you see on the FTP server
-through rclone are those of upload.
-
 Rclone's FTP backend does not support any checksums but can compare
 file sizes.
 
@@ -324,3 +330,23 @@ Rclone's FTP backend could support server-side move but does not
 at present.
 
 The `ftp_proxy` environment variable is not currently supported.
+
+#### Modified time
+
+File modification time (timestamps) is supported to 1 second resolution
+for major FTP servers: ProFTPd, PureFTPd, VsFTPd, and FileZilla FTP server.
+The `VsFTPd` server has non-standard implementation of time related protocol
+commands and needs a special configuration setting: `writing_mdtm = true`.
+
+Support for precise file time with other FTP servers varies depending on what
+protocol extensions they advertise. If all the `MLSD`, `MDTM` and `MFTM`
+extensions are present, rclone will use them together to provide precise time.
+Otherwise the times you see on the FTP server through rclone are those of the
+last file upload.
+
+You can use the following command to check whether rclone can use precise time
+with your FTP server: `rclone backend features your_ftp_remote:` (the trailing
+colon is important). Look for the number in the line tagged by `Precision`
+designating the remote time precision expressed as nanoseconds. A value of
+`1000000000` means that file time precision of 1 second is available.
+A value of `3153600000000000000` (or another large number) means "unsupported".

fstest/testserver/init.d/TestFTPVsftpd

@@ -18,6 +18,7 @@ start() {
     echo host=$(docker_ip)
     echo user=$USER
     echo pass=$(rclone obscure $PASS)
+    echo writing_mdtm=true
     echo encoding=Ctl,LeftPeriod,Slash
     echo _connect=$(docker_ip):21
 }