cmd/serve: add ftp server - implement #2151

Makefile

@@ -229,4 +229,3 @@ startdev:
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe
-

cmd/serve/ftp/ftp.go

@@ -0,0 +1,416 @@
+package ftp
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/user"
+	"strconv"
+	"sync"
+
+	ftp "github.com/goftp/server"
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/cmd/serve/ftp/ftpflags"
+	"github.com/ncw/rclone/cmd/serve/ftp/ftpopt"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/accounting"
+	"github.com/ncw/rclone/fs/log"
+	"github.com/ncw/rclone/vfs"
+	"github.com/ncw/rclone/vfs/vfsflags"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	ftpflags.AddFlags(Command.Flags())
+	vfsflags.AddFlags(Command.Flags())
+}
+
+// Command definition for cobra
+var Command = &cobra.Command{
+	Use:   "ftp remote:path",
+	Short: `Serve remote:path over FTP.`,
+	Long: `
+rclone serve ftp implements a basic ftp server to serve the
+remote over FTP protocol. This can be viewed with a ftp client
+or you can make a remote of type ftp to read and write it.
+` + ftpopt.Help + vfs.Help,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1, command, args)
+		f := cmd.NewFsSrc(args)
+		cmd.Run(false, false, command, func() error {
+			s, err := newServer(f, &ftpflags.Opt)
+			if err != nil {
+				return err
+			}
+			return s.serve()
+		})
+	},
+}
+
+// server contains everything to run the server
+type server struct {
+	f   fs.Fs
+	srv *ftp.Server
+}
+
+// Make a new FTP to serve the remote
+func newServer(f fs.Fs, opt *ftpopt.Options) (*server, error) {
+	host, port, err := net.SplitHostPort(opt.ListenAddr)
+	if err != nil {
+		return nil, errors.New("Failed to parse host:port")
+	}
+	portNum, err := strconv.Atoi(port)
+	if err != nil {
+		return nil, errors.New("Failed to parse host:port")
+	}
+
+	ftpopt := &ftp.ServerOpts{
+		Name:           "Rclone FTP Server",
+		WelcomeMessage: "Welcome on Rclone FTP Server",
+		Factory: &DriverFactory{
+			vfs: vfs.New(f, &vfsflags.Opt),
+		},
+		Hostname:     host,
+		Port:         portNum,
+		PassivePorts: opt.PassivePorts,
+		Auth: &Auth{
+			BasicUser: opt.BasicUser,
+			BasicPass: opt.BasicPass,
+		},
+		Logger: &Logger{},
+		//TODO implement a maximum of https://godoc.org/github.com/goftp/server#ServerOpts
+	}
+	return &server{
+		f:   f,
+		srv: ftp.NewServer(ftpopt),
+	}, nil
+}
+
+// serve runs the ftp server
+func (s *server) serve() error {
+	fs.Logf(s.f, "Serving FTP on %s", s.srv.Hostname+":"+strconv.Itoa(s.srv.Port))
+	return s.srv.ListenAndServe()
+}
+
+// serve runs the ftp server
+func (s *server) close() error {
+	fs.Logf(s.f, "Stopping FTP on %s", s.srv.Hostname+":"+strconv.Itoa(s.srv.Port))
+	return s.srv.Shutdown()
+}
+
+//Logger ftp logger output formatted message
+type Logger struct{}
+
+//Print log simple text message
+func (l *Logger) Print(sessionID string, message interface{}) {
+	fs.Infof(sessionID, "%s", message)
+}
+
+//Printf log formatted text message
+func (l *Logger) Printf(sessionID string, format string, v ...interface{}) {
+	fs.Infof(sessionID, format, v...)
+}
+
+//PrintCommand log formatted command execution
+func (l *Logger) PrintCommand(sessionID string, command string, params string) {
+	if command == "PASS" {
+		fs.Infof(sessionID, "> PASS ****")
+	} else {
+		fs.Infof(sessionID, "> %s %s", command, params)
+	}
+}
+
+//PrintResponse log responses
+func (l *Logger) PrintResponse(sessionID string, code int, message string) {
+	fs.Infof(sessionID, "< %d %s", code, message)
+}
+
+//Auth struct to handle ftp auth (temporary simple for POC)
+type Auth struct {
+	BasicUser string
+	BasicPass string
+}
+
+//CheckPasswd handle auth based on configuration
+func (a *Auth) CheckPasswd(user, pass string) (bool, error) {
+	return a.BasicUser == user && (a.BasicPass == "" || a.BasicPass == pass), nil
+}
+
+//DriverFactory factory of ftp driver for each session
+type DriverFactory struct {
+	vfs *vfs.VFS
+}
+
+//NewDriver start a new session
+func (f *DriverFactory) NewDriver() (ftp.Driver, error) {
+	log.Trace("", "Init driver")("")
+	return &Driver{
+		vfs: f.vfs,
+	}, nil
+}
+
+//Driver impletation of ftp server
+type Driver struct {
+	vfs  *vfs.VFS
+	lock sync.Mutex
+}
+
+//Init a connection
+func (d *Driver) Init(*ftp.Conn) {
+	defer log.Trace("", "Init session")("")
+}
+
+//Stat get information on file or folder
+func (d *Driver) Stat(path string) (fi ftp.FileInfo, err error) {
+	defer log.Trace(path, "")("fi=%+v, err = %v", &fi, &err)
+	n, err := d.vfs.Stat(path)
+	if err != nil {
+		return nil, err
+	}
+	return &FileInfo{n, n.Mode(), d.vfs.Opt.UID, d.vfs.Opt.GID}, err
+}
+
+//ChangeDir move current folder
+func (d *Driver) ChangeDir(path string) (err error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	defer log.Trace(path, "")("err = %v", &err)
+	n, err := d.vfs.Stat(path)
+	if err != nil {
+		return err
+	}
+	if !n.IsDir() {
+		return errors.New("Not a directory")
+	}
+	return nil
+}
+
+//ListDir list content of a folder
+func (d *Driver) ListDir(path string, callback func(ftp.FileInfo) error) (err error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	defer log.Trace(path, "")("err = %v", &err)
+	node, err := d.vfs.Stat(path)
+	if err == vfs.ENOENT {
+		return errors.New("Directory not found")
+	} else if err != nil {
+		return err
+	}
+	if !node.IsDir() {
+		return errors.New("Not a directory")
+	}
+
+	dir := node.(*vfs.Dir)
+	dirEntries, err := dir.ReadDirAll()
+	if err != nil {
+		return err
+	}
+
+	// Account the transfer
+	accounting.Stats.Transferring(path)
+	defer accounting.Stats.DoneTransferring(path, true)
+
+	for _, file := range dirEntries {
+		err = callback(&FileInfo{file, file.Mode(), d.vfs.Opt.UID, d.vfs.Opt.GID})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+//DeleteDir delete a folder and his content
+func (d *Driver) DeleteDir(path string) (err error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	defer log.Trace(path, "")("err = %v", &err)
+	node, err := d.vfs.Stat(path)
+	if err != nil {
+		return err
+	}
+	if !node.IsDir() {
+		return errors.New("Not a directory")
+	}
+	err = node.Remove()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+//DeleteFile delete a file
+func (d *Driver) DeleteFile(path string) (err error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	defer log.Trace(path, "")("err = %v", &err)
+	node, err := d.vfs.Stat(path)
+	if err != nil {
+		return err
+	}
+	if !node.IsFile() {
+		return errors.New("Not a file")
+	}
+	err = node.Remove()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+//Rename rename a file or folder
+func (d *Driver) Rename(oldName, newName string) (err error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	defer log.Trace(oldName, "newName=%q", newName)("err = %v", &err)
+	return d.vfs.Rename(oldName, newName)
+}
+
+//MakeDir create a folder
+func (d *Driver) MakeDir(path string) (err error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	defer log.Trace(path, "")("err = %v", &err)
+	dir, leaf, err := d.vfs.StatParent(path)
+	if err != nil {
+		return err
+	}
+	_, err = dir.Mkdir(leaf)
+	return err
+}
+
+//GetFile download a file
+func (d *Driver) GetFile(path string, offset int64) (size int64, fr io.ReadCloser, err error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	defer log.Trace(path, "offset=%v", offset)("err = %v", &err)
+	node, err := d.vfs.Stat(path)
+	if err == vfs.ENOENT {
+		fs.Infof(path, "File not found")
+		return 0, nil, errors.New("File not found")
+	} else if err != nil {
+		return 0, nil, err
+	}
+	if !node.IsFile() {
+		return 0, nil, errors.New("Not a file")
+	}
+
+	handle, err := node.Open(os.O_RDONLY)
+	if err != nil {
+		return 0, nil, err
+	}
+	_, err = handle.Seek(offset, os.SEEK_SET)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	// Account the transfer
+	accounting.Stats.Transferring(path)
+	defer accounting.Stats.DoneTransferring(path, true)
+
+	return node.Size(), handle, nil
+}
+
+//PutFile upload a file
+func (d *Driver) PutFile(path string, data io.Reader, appendData bool) (n int64, err error) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	defer log.Trace(path, "append=%v", appendData)("err = %v", &err)
+	var isExist bool
+	node, err := d.vfs.Stat(path)
+	if err == nil {
+		isExist = true
+		if node.IsDir() {
+			return 0, errors.New("A dir has the same name")
+		}
+	} else {
+		if os.IsNotExist(err) {
+			isExist = false
+		} else {
+			return 0, err
+		}
+	}
+
+	if appendData && !isExist {
+		appendData = false
+	}
+
+	if !appendData {
+		if isExist {
+			err = node.Remove()
+			if err != nil {
+				return 0, err
+			}
+		}
+		f, err := d.vfs.OpenFile(path, os.O_RDWR|os.O_CREATE, 0660)
+		if err != nil {
+			return 0, err
+		}
+		defer closeIO(path, f)
+		bytes, err := io.Copy(f, data)
+		if err != nil {
+			return 0, err
+		}
+		return bytes, nil
+	}
+
+	of, err := d.vfs.OpenFile(path, os.O_APPEND|os.O_RDWR, 0660)
+	if err != nil {
+		return 0, err
+	}
+	defer closeIO(path, of)
+
+	_, err = of.Seek(0, os.SEEK_END)
+	if err != nil {
+		return 0, err
+	}
+
+	bytes, err := io.Copy(of, data)
+	if err != nil {
+		return 0, err
+	}
+
+	return bytes, nil
+}
+
+//FileInfo  struct ot hold file infor for ftp server
+type FileInfo struct {
+	os.FileInfo
+
+	mode  os.FileMode
+	owner uint32
+	group uint32
+}
+
+//Mode return êrm mode of file.
+func (f *FileInfo) Mode() os.FileMode {
+	return f.mode
+}
+
+//Owner return owner of file. Try to find the username if possible
+func (f *FileInfo) Owner() string {
+	str := fmt.Sprint(f.owner)
+	u, err := user.LookupId(str)
+	if err != nil {
+		return str //User not found
+	}
+	return u.Username
+}
+
+//Group return group of file. Try to find the group name if possible
+func (f *FileInfo) Group() string {
+	str := fmt.Sprint(f.group)
+	g, err := user.LookupGroupId(str)
+	if err != nil {
+		return str //Group not found default to numrical value
+	}
+	return g.Name
+}
+
+func closeIO(path string, c io.Closer) {
+	err := c.Close()
+	if err != nil {
+		log.Trace(path, "")("err = %v", &err)
+	}
+}

cmd/serve/ftp/ftp_test.go

@@ -0,0 +1,89 @@
+// Serve ftp tests set up a server and run the integration tests
+// for the ftp remote against it.
+//
+// We skip tests on platforms with troublesome character mappings
+
+//+build !windows,!darwin
+
+package ftp
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+
+	ftp "github.com/goftp/server"
+	_ "github.com/ncw/rclone/backend/local"
+	"github.com/ncw/rclone/cmd/serve/ftp/ftpopt"
+	"github.com/ncw/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testHOST             = "localhost"
+	testPORT             = "51780"
+	testPASSIVEPORTRANGE = "30000-32000"
+)
+
+// TestFTP runs the ftp server then runs the unit tests for the
+// ftp remote against it.
+func TestFTP(t *testing.T) {
+	opt := ftpopt.DefaultOpt
+	opt.ListenAddr = testHOST + ":" + testPORT
+	opt.PassivePorts = testPASSIVEPORTRANGE
+	opt.BasicUser = "rclone"
+	opt.BasicPass = "password"
+
+	fstest.Initialise()
+
+	fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
+	assert.NoError(t, err)
+	defer clean()
+
+	err = fremote.Mkdir("")
+	assert.NoError(t, err)
+
+	// Start the server
+	w, err := newServer(fremote, &opt)
+	assert.NoError(t, err)
+
+	go func() {
+		err := w.serve()
+		if err != ftp.ErrServerClosed {
+			assert.NoError(t, err)
+		}
+	}()
+	defer func() {
+		err := w.close()
+		assert.NoError(t, err)
+	}()
+
+	// Change directory to run the tests
+	err = os.Chdir("../../../backend/ftp")
+	assert.NoError(t, err, "failed to cd to ftp remote")
+
+	// Run the ftp tests with an on the fly remote
+	args := []string{"test"}
+	if testing.Verbose() {
+		args = append(args, "-v")
+	}
+	if *fstest.Verbose {
+		args = append(args, "-verbose")
+	}
+	args = append(args, "-list-retries", fmt.Sprint(*fstest.ListRetries))
+	args = append(args, "-remote", "ftptest:")
+	cmd := exec.Command("go", args...)
+	cmd.Env = append(os.Environ(),
+		"RCLONE_CONFIG_FTPTEST_TYPE=ftp",
+		"RCLONE_CONFIG_FTPTEST_HOST="+testHOST,
+		"RCLONE_CONFIG_FTPTEST_PORT="+testPORT,
+		"RCLONE_CONFIG_FTPTEST_USER=rclone",
+		"RCLONE_CONFIG_FTPTEST_PASS=0HU5Hx42YiLoNGJxppOOP3QTbr-KB_MP", // ./rclone obscure password
+	)
+	out, err := cmd.CombinedOutput()
+	if len(out) != 0 {
+		t.Logf("\n----------\n%s----------\n", string(out))
+	}
+	assert.NoError(t, err, "Running ftp integration tests")
+}

cmd/serve/ftp/ftpflags/ftpflags.go

@@ -0,0 +1,25 @@
+package ftpflags
+
+import (
+	"github.com/ncw/rclone/cmd/serve/ftp/ftpopt"
+	"github.com/ncw/rclone/fs/config/flags"
+	"github.com/spf13/pflag"
+)
+
+// Options set by command line flags
+var (
+	Opt = ftpopt.DefaultOpt
+)
+
+// AddFlagsPrefix adds flags for the ftpopt
+func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *ftpopt.Options) {
+	flags.StringVarP(flagSet, &Opt.ListenAddr, prefix+"addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to.")
+	flags.StringVarP(flagSet, &Opt.PassivePorts, prefix+"passive-port", "", Opt.PassivePorts, "Passive port range to use.")
+	flags.StringVarP(flagSet, &Opt.BasicUser, prefix+"user", "", Opt.BasicUser, "User name for authentication.")
+	flags.StringVarP(flagSet, &Opt.BasicPass, prefix+"pass", "", Opt.BasicPass, "Password for authentication. (empty value allow every password)")
+}
+
+// AddFlags adds flags for the httplib
+func AddFlags(flagSet *pflag.FlagSet) {
+	AddFlagsPrefix(flagSet, "", &Opt)
+}

cmd/serve/ftp/ftpopt/ftp_options.go

@@ -0,0 +1,38 @@
+package ftpopt
+
+// Help contains text describing the http server to add to the command
+// help.
+var Help = `
+### Server options
+
+Use --addr to specify which IP address and port the server should
+listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all
+IPs.  By default it only listens on localhost.  You can use port
+:0 to let the OS choose an available port.
+
+If you set --addr to listen on a public or LAN accessible IP address
+then using Authentication is advised - see the next section for info.
+
+#### Authentication
+
+By default this will serve files without needing a login.
+
+You can set a single username and password with the --user and --pass flags.
+`
+
+// Options contains options for the http Server
+type Options struct {
+	//TODO add more options
+	ListenAddr   string // Port to listen on
+	PassivePorts string // Passive ports range
+	BasicUser    string // single username for basic auth if not using Htpasswd
+	BasicPass    string // password for BasicUser
+}
+
+// DefaultOpt is the default values used for Options
+var DefaultOpt = Options{
+	ListenAddr:   "localhost:2121",
+	PassivePorts: "30000-32000",
+	BasicUser:    "anonymous",
+	BasicPass:    "",
+}

cmd/serve/serve.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 
 	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/cmd/serve/ftp"
 	"github.com/ncw/rclone/cmd/serve/http"
 	"github.com/ncw/rclone/cmd/serve/restic"
 	"github.com/ncw/rclone/cmd/serve/webdav"
@@ -14,6 +15,7 @@ func init() {
 	Command.AddCommand(http.Command)
 	Command.AddCommand(webdav.Command)
 	Command.AddCommand(restic.Command)
+	Command.AddCommand(ftp.Command)
 	cmd.Root.AddCommand(Command)
 }