// Package koofr provides an interface to the Koofr storage system. package koofr import ( "context" "encoding/base64" "errors" "fmt" "io" "net/http" "path" "strings" "time" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/encoder" httpclient "github.com/koofr/go-httpclient" koofrclient "github.com/koofr/go-koofrclient" ) // Register Fs with rclone func init() { fs.Register(&fs.RegInfo{ Name: "koofr", Description: "Koofr, Digi Storage and other Koofr-compatible storage providers", NewFs: NewFs, Options: []fs.Option{{ Name: fs.ConfigProvider, Help: "Choose your storage provider.", // NOTE if you add a new provider here, then add it in the // setProviderDefaults() function and update options accordingly Examples: []fs.OptionExample{{ Value: "koofr", Help: "Koofr, https://app.koofr.net/", }, { Value: "digistorage", Help: "Digi Storage, https://storage.rcs-rds.ro/", }, { Value: "other", Help: "Any other Koofr API compatible storage service", }}, }, { Name: "endpoint", Help: "The Koofr API endpoint to use.", Provider: "other", Required: true, }, { Name: "mountid", Help: "Mount ID of the mount to use.\n\nIf omitted, the primary mount is used.", Advanced: true, }, { Name: "setmtime", Help: "Does the backend support setting modification time.\n\nSet this to false if you use a mount ID that points to a Dropbox or Amazon Drive backend.", Default: true, Advanced: true, }, { Name: "user", Help: "Your user name.", Required: true, Sensitive: true, }, { Name: "password", Help: "Your password for rclone generate one at https://app.koofr.net/app/admin/preferences/password.", Provider: "koofr", IsPassword: true, Required: true, }, { Name: "password", Help: "Your password for rclone generate one at https://storage.rcs-rds.ro/app/admin/preferences/password.", Provider: "digistorage", IsPassword: true, Required: true, }, { Name: "password", Help: "Your password for rclone (generate one at your service's settings page).", Provider: "other", IsPassword: true, Required: true, }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, // Encode invalid UTF-8 bytes as json doesn't handle them properly. Default: (encoder.Display | encoder.EncodeBackSlash | encoder.EncodeInvalidUtf8), }}, }) } // Options represent the configuration of the Koofr backend type Options struct { Provider string `config:"provider"` Endpoint string `config:"endpoint"` MountID string `config:"mountid"` User string `config:"user"` Password string `config:"password"` SetMTime bool `config:"setmtime"` Enc encoder.MultiEncoder `config:"encoding"` } // An Fs is a representation of a remote Koofr Fs type Fs struct { name string mountID string root string opt Options features *fs.Features client *koofrclient.KoofrClient } // An Object on the remote Koofr Fs type Object struct { fs *Fs remote string info koofrclient.FileInfo } func base(pth string) string { rv := path.Base(pth) if rv == "" || rv == "." { rv = "/" } return rv } func dir(pth string) string { rv := path.Dir(pth) if rv == "" || rv == "." { rv = "/" } return rv } // String returns a string representation of the remote Object func (o *Object) String() string { return o.remote } // Remote returns the remote path of the Object, relative to Fs root func (o *Object) Remote() string { return o.remote } // ModTime returns the modification time of the Object func (o *Object) ModTime(ctx context.Context) time.Time { return time.Unix(o.info.Modified/1000, (o.info.Modified%1000)*1000*1000) } // Size return the size of the Object in bytes func (o *Object) Size() int64 { return o.info.Size } // Fs returns a reference to the Koofr Fs containing the Object func (o *Object) Fs() fs.Info { return o.fs } // Hash returns an MD5 hash of the Object func (o *Object) Hash(ctx context.Context, typ hash.Type) (string, error) { if typ == hash.MD5 { return o.info.Hash, nil } return "", nil } // fullPath returns full path of the remote Object (including Fs root) func (o *Object) fullPath() string { return o.fs.fullPath(o.remote) } // Storable returns true if the Object is storable func (o *Object) Storable() bool { return true } // SetModTime is not supported func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error { return fs.ErrorCantSetModTimeWithoutDelete } // Open opens the Object for reading func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { var sOff, eOff int64 = 0, -1 fs.FixRangeOption(options, o.Size()) for _, option := range options { switch x := option.(type) { case *fs.SeekOption: sOff = x.Offset case *fs.RangeOption: sOff = x.Start eOff = x.End default: if option.Mandatory() { fs.Logf(o, "Unsupported mandatory option: %v", option) } } } if sOff == 0 && eOff < 0 { return o.fs.client.FilesGet(o.fs.mountID, o.fullPath()) } span := &koofrclient.FileSpan{ Start: sOff, End: eOff, } return o.fs.client.FilesGetRange(o.fs.mountID, o.fullPath(), span) } // Update updates the Object contents func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { mtime := src.ModTime(ctx).UnixNano() / 1000 / 1000 putopts := &koofrclient.PutOptions{ ForceOverwrite: true, NoRename: true, OverwriteIgnoreNonExisting: true, SetModified: &mtime, } fullPath := o.fullPath() dirPath := dir(fullPath) name := base(fullPath) err := o.fs.mkdir(dirPath) if err != nil { return err } info, err := o.fs.client.FilesPutWithOptions(o.fs.mountID, dirPath, name, in, putopts) if err != nil { return err } o.info = *info return nil } // Remove deletes the remote Object func (o *Object) Remove(ctx context.Context) error { return o.fs.client.FilesDelete(o.fs.mountID, o.fullPath()) } // Name returns the name of the Fs func (f *Fs) Name() string { return f.name } // Root returns the root path of the Fs func (f *Fs) Root() string { return f.root } // String returns a string representation of the Fs func (f *Fs) String() string { return "koofr:" + f.mountID + ":" + f.root } // Features returns the optional features supported by this Fs func (f *Fs) Features() *fs.Features { return f.features } // Precision denotes that setting modification times is not supported func (f *Fs) Precision() time.Duration { if !f.opt.SetMTime { return fs.ModTimeNotSupported } return time.Millisecond } // Hashes returns a set of hashes are Provided by the Fs func (f *Fs) Hashes() hash.Set { return hash.Set(hash.MD5) } // fullPath constructs a full, absolute path from an Fs root relative path, func (f *Fs) fullPath(part string) string { return f.opt.Enc.FromStandardPath(path.Join("/", f.root, part)) } func setProviderDefaults(opt *Options) { // handle old, provider-less configs if opt.Provider == "" { if opt.Endpoint == "" || strings.HasPrefix(opt.Endpoint, "https://app.koofr.net") { opt.Provider = "koofr" } else if strings.HasPrefix(opt.Endpoint, "https://storage.rcs-rds.ro") { opt.Provider = "digistorage" } else { opt.Provider = "other" } } // now assign an endpoint if opt.Provider == "koofr" { opt.Endpoint = "https://app.koofr.net" } else if opt.Provider == "digistorage" { opt.Endpoint = "https://storage.rcs-rds.ro" } } // NewFs constructs a new filesystem given a root path and rclone configuration options func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) { opt := new(Options) err = configstruct.Set(m, opt) if err != nil { return nil, err } setProviderDefaults(opt) return NewFsFromOptions(ctx, name, root, opt) } // NewFsFromOptions constructs a new filesystem given a root path and internal configuration options func NewFsFromOptions(ctx context.Context, name, root string, opt *Options) (ff fs.Fs, err error) { pass, err := obscure.Reveal(opt.Password) if err != nil { return nil, err } httpClient := httpclient.New() httpClient.Client = fshttp.NewClient(ctx) client := koofrclient.NewKoofrClientWithHTTPClient(opt.Endpoint, httpClient) basicAuth := fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(opt.User+":"+pass))) client.HTTPClient.Headers.Set("Authorization", basicAuth) mounts, err := client.Mounts() if err != nil { return nil, err } f := &Fs{ name: name, root: root, opt: *opt, client: client, } f.features = (&fs.Features{ CaseInsensitive: true, DuplicateFiles: false, BucketBased: false, CanHaveEmptyDirectories: true, }).Fill(ctx, f) for _, m := range mounts { if opt.MountID != "" { if m.Id == opt.MountID { f.mountID = m.Id break } } else if m.IsPrimary { f.mountID = m.Id break } } if f.mountID == "" { if opt.MountID == "" { return nil, errors.New("failed to find primary mount") } return nil, errors.New("failed to find mount " + opt.MountID) } rootFile, err := f.client.FilesInfo(f.mountID, f.opt.Enc.FromStandardPath("/"+f.root)) if err == nil && rootFile.Type != "dir" { f.root = dir(f.root) err = fs.ErrorIsFile } else { err = nil } return f, err } // List returns a list of items in a directory func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { files, err := f.client.FilesList(f.mountID, f.fullPath(dir)) if err != nil { return nil, translateErrorsDir(err) } entries = make([]fs.DirEntry, len(files)) for i, file := range files { remote := path.Join(dir, f.opt.Enc.ToStandardName(file.Name)) if file.Type == "dir" { entries[i] = fs.NewDir(remote, time.Time{}) } else { entries[i] = &Object{ fs: f, info: file, remote: remote, } } } return entries, nil } // NewObject creates a new remote Object for a given remote path func (f *Fs) NewObject(ctx context.Context, remote string) (obj fs.Object, err error) { info, err := f.client.FilesInfo(f.mountID, f.fullPath(remote)) if err != nil { return nil, translateErrorsObject(err) } if info.Type == "dir" { return nil, fs.ErrorIsDir } return &Object{ fs: f, info: info, remote: remote, }, nil } // Put updates a remote Object func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (obj fs.Object, err error) { mtime := src.ModTime(ctx).UnixNano() / 1000 / 1000 putopts := &koofrclient.PutOptions{ ForceOverwrite: true, NoRename: true, OverwriteIgnoreNonExisting: true, SetModified: &mtime, } fullPath := f.fullPath(src.Remote()) dirPath := dir(fullPath) name := base(fullPath) err = f.mkdir(dirPath) if err != nil { return nil, err } info, err := f.client.FilesPutWithOptions(f.mountID, dirPath, name, in, putopts) if err != nil { return nil, translateErrorsObject(err) } return &Object{ fs: f, info: *info, remote: src.Remote(), }, nil } // PutStream updates a remote Object with a stream of unknown size func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { return f.Put(ctx, in, src, options...) } // isBadRequest is a predicate which holds true iff the error returned was // HTTP status 400 func isBadRequest(err error) bool { switch err := err.(type) { case httpclient.InvalidStatusError: if err.Got == http.StatusBadRequest { return true } } return false } // translateErrorsDir translates koofr errors to rclone errors (for a dir // operation) func translateErrorsDir(err error) error { switch err := err.(type) { case httpclient.InvalidStatusError: if err.Got == http.StatusNotFound { return fs.ErrorDirNotFound } } return err } // translatesErrorsObject translates Koofr errors to rclone errors (for an object operation) func translateErrorsObject(err error) error { switch err := err.(type) { case httpclient.InvalidStatusError: if err.Got == http.StatusNotFound { return fs.ErrorObjectNotFound } } return err } // mkdir creates a directory at the given remote path. Creates ancestors if // necessary func (f *Fs) mkdir(fullPath string) error { if fullPath == "/" { return nil } info, err := f.client.FilesInfo(f.mountID, fullPath) if err == nil && info.Type == "dir" { return nil } err = translateErrorsDir(err) if err != nil && err != fs.ErrorDirNotFound { return err } dirs := strings.Split(fullPath, "/") parent := "/" for _, part := range dirs { if part == "" { continue } info, err = f.client.FilesInfo(f.mountID, path.Join(parent, part)) if err != nil || info.Type != "dir" { err = translateErrorsDir(err) if err != nil && err != fs.ErrorDirNotFound { return err } err = f.client.FilesNewFolder(f.mountID, parent, part) if err != nil && !isBadRequest(err) { return err } } parent = path.Join(parent, part) } return nil } // Mkdir creates a directory at the given remote path. Creates ancestors if // necessary func (f *Fs) Mkdir(ctx context.Context, dir string) error { fullPath := f.fullPath(dir) return f.mkdir(fullPath) } // Rmdir removes an (empty) directory at the given remote path func (f *Fs) Rmdir(ctx context.Context, dir string) error { files, err := f.client.FilesList(f.mountID, f.fullPath(dir)) if err != nil { return translateErrorsDir(err) } if len(files) > 0 { return fs.ErrorDirectoryNotEmpty } err = f.client.FilesDelete(f.mountID, f.fullPath(dir)) if err != nil { return translateErrorsDir(err) } return nil } // Copy copies a remote Object to the given path func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { dstFullPath := f.fullPath(remote) dstDir := dir(dstFullPath) err := f.mkdir(dstDir) if err != nil { return nil, fs.ErrorCantCopy } mtime := src.ModTime(ctx).UnixNano() / 1000 / 1000 err = f.client.FilesCopy((src.(*Object)).fs.mountID, (src.(*Object)).fs.fullPath((src.(*Object)).remote), f.mountID, dstFullPath, koofrclient.CopyOptions{SetModified: &mtime}) if err != nil { return nil, fs.ErrorCantCopy } return f.NewObject(ctx, remote) } // Move moves a remote Object to the given path func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { srcObj := src.(*Object) dstFullPath := f.fullPath(remote) dstDir := dir(dstFullPath) err := f.mkdir(dstDir) if err != nil { return nil, fs.ErrorCantMove } err = f.client.FilesMove(srcObj.fs.mountID, srcObj.fs.fullPath(srcObj.remote), f.mountID, dstFullPath) if err != nil { return nil, fs.ErrorCantMove } return f.NewObject(ctx, remote) } // DirMove moves a remote directory to the given path func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { srcFs := src.(*Fs) srcFullPath := srcFs.fullPath(srcRemote) dstFullPath := f.fullPath(dstRemote) if srcFs.mountID == f.mountID && srcFullPath == dstFullPath { return fs.ErrorDirExists } dstDir := dir(dstFullPath) err := f.mkdir(dstDir) if err != nil { return fs.ErrorCantDirMove } err = f.client.FilesMove(srcFs.mountID, srcFullPath, f.mountID, dstFullPath) if err != nil { return fs.ErrorCantDirMove } return nil } // About reports space usage (with a MiB precision) func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { mount, err := f.client.MountsDetails(f.mountID) if err != nil { return nil, err } return &fs.Usage{ Total: fs.NewUsageValue(mount.SpaceTotal * 1024 * 1024), Used: fs.NewUsageValue(mount.SpaceUsed * 1024 * 1024), Trashed: nil, Other: nil, Free: fs.NewUsageValue((mount.SpaceTotal - mount.SpaceUsed) * 1024 * 1024), Objects: nil, }, nil } // Purge purges the complete Fs func (f *Fs) Purge(ctx context.Context) error { err := translateErrorsDir(f.client.FilesDelete(f.mountID, f.fullPath(""))) return err } // linkCreate is a Koofr API request for creating a public link type linkCreate struct { Path string `json:"path"` } // link is a Koofr API response to creating a public link type link struct { ID string `json:"id"` Name string `json:"name"` Path string `json:"path"` Counter int64 `json:"counter"` URL string `json:"url"` ShortURL string `json:"shortUrl"` Hash string `json:"hash"` Host string `json:"host"` HasPassword bool `json:"hasPassword"` Password string `json:"password"` ValidFrom int64 `json:"validFrom"` ValidTo int64 `json:"validTo"` PasswordRequired bool `json:"passwordRequired"` } // createLink makes a Koofr API call to create a public link func createLink(c *koofrclient.KoofrClient, mountID string, path string) (*link, error) { linkCreate := linkCreate{ Path: path, } linkData := link{} request := httpclient.RequestData{ Method: "POST", Path: "/api/v2/mounts/" + mountID + "/links", ExpectedStatus: []int{http.StatusOK, http.StatusCreated}, ReqEncoding: httpclient.EncodingJSON, ReqValue: linkCreate, RespEncoding: httpclient.EncodingJSON, RespValue: &linkData, } _, err := c.Request(&request) if err != nil { return nil, err } return &linkData, nil } // PublicLink creates a public link to the remote path func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) { linkData, err := createLink(f.client, f.mountID, f.fullPath(remote)) if err != nil { return "", translateErrorsDir(err) } // URL returned by API looks like following: // // https://app.koofr.net/links/35d9fb92-74a3-4930-b4ed-57f123bfb1a6 // // Direct url looks like following: // // https://app.koofr.net/content/links/39a6cc01-3b23-477a-8059-c0fb3b0f15de/files/get?path=%2F // // I am not sure about meaning of "path" parameter; in my experiments // it is always "%2F", and omitting it or putting any other value // results in 404. // // There is one more quirk: direct link to file in / returns that file, // direct link to file somewhere else in hierarchy returns zip archive // with one member. link := linkData.URL link = strings.ReplaceAll(link, "/links", "/content/links") link += "/files/get?path=%2F" return link, nil }