2017-10-02 21:29:23 +02:00
// Package webdav provides an interface to the Webdav
// object storage system.
package webdav
// SetModTime might be possible
// https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file
// ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option.
// For example the ownCloud WebDAV server does it that way.
import (
2019-01-27 14:33:21 +01:00
"bytes"
2019-06-17 10:34:30 +02:00
"context"
2021-02-06 15:50:53 +01:00
"crypto/tls"
2018-05-10 16:02:41 +02:00
"encoding/xml"
2021-11-04 11:12:57 +01:00
"errors"
2017-10-02 21:29:23 +02:00
"fmt"
"io"
"net/http"
"net/url"
2018-09-04 15:04:13 +02:00
"os/exec"
2017-10-02 21:29:23 +02:00
"path"
2022-05-20 11:06:55 +02:00
"regexp"
2020-06-14 13:11:19 +02:00
"strconv"
2017-10-02 21:29:23 +02:00
"strings"
2021-02-22 02:07:05 +01:00
"sync"
2017-10-02 21:29:23 +02:00
"time"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/backend/webdav/api"
"github.com/rclone/rclone/backend/webdav/odrvcookie"
"github.com/rclone/rclone/fs"
2021-02-21 16:53:06 +01:00
"github.com/rclone/rclone/fs/config"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
2021-02-21 16:53:06 +01:00
"github.com/rclone/rclone/lib/encoder"
2019-07-28 19:47:38 +02:00
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
2019-01-17 13:35:30 +01:00
ntlmssp "github.com/Azure/go-ntlmssp"
2017-10-02 21:29:23 +02:00
)
const (
2023-03-26 22:36:48 +02:00
minSleep = fs . Duration ( 10 * time . Millisecond )
2017-10-02 21:29:23 +02:00
maxSleep = 2 * time . Second
2018-08-04 12:02:47 +02:00
decayConstant = 2 // bigger for slower decay, exponential
defaultDepth = "1" // depth for PROPFIND
2017-10-02 21:29:23 +02:00
)
2021-02-21 16:53:06 +01:00
const defaultEncodingSharepointNTLM = ( encoder . EncodeWin |
encoder . EncodeHashPercent | // required by IIS/8.5 in contrast with onedrive which doesn't need it
( encoder . Display &^ encoder . EncodeDot ) | // test with IIS/8.5 shows that EncodeDot is not needed
encoder . EncodeBackSlash |
encoder . EncodeLeftSpace |
encoder . EncodeLeftTilde |
encoder . EncodeRightPeriod |
encoder . EncodeRightSpace |
encoder . EncodeInvalidUtf8 )
2017-10-02 21:29:23 +02:00
// Register with Fs
func init ( ) {
2021-02-21 16:53:06 +01:00
configEncodingHelp := fmt . Sprintf (
"%s\n\nDefault encoding is %s for sharepoint-ntlm or identity otherwise." ,
config . ConfigEncodingHelp , defaultEncodingSharepointNTLM )
2017-10-02 21:29:23 +02:00
fs . Register ( & fs . RegInfo {
Name : "webdav" ,
2022-06-19 18:59:54 +02:00
Description : "WebDAV" ,
2017-10-02 21:29:23 +02:00
NewFs : NewFs ,
Options : [ ] fs . Option { {
Name : "url" ,
2021-08-22 15:11:41 +02:00
Help : "URL of http host to connect to.\n\nE.g. https://example.com." ,
2018-05-14 19:06:57 +02:00
Required : true ,
2017-10-02 21:29:23 +02:00
} , {
2018-05-14 19:06:57 +02:00
Name : "vendor" ,
2022-06-19 18:59:54 +02:00
Help : "Name of the WebDAV site/service/software you are using." ,
2017-10-02 21:29:23 +02:00
Examples : [ ] fs . OptionExample { {
2023-03-13 05:45:54 +01:00
Value : "fastmail" ,
Help : "Fastmail Files" ,
} , {
2017-10-02 21:29:23 +02:00
Value : "nextcloud" ,
Help : "Nextcloud" ,
} , {
Value : "owncloud" ,
Help : "Owncloud" ,
2018-04-09 10:05:43 +02:00
} , {
Value : "sharepoint" ,
2021-08-16 11:30:01 +02:00
Help : "Sharepoint Online, authenticated by Microsoft account" ,
2019-01-17 13:35:30 +01:00
} , {
Value : "sharepoint-ntlm" ,
2021-08-16 11:30:01 +02:00
Help : "Sharepoint with NTLM authentication, usually self-hosted or on-premises" ,
2023-11-05 13:37:25 +01:00
} , {
Value : "rclone" ,
Help : "rclone WebDAV server to serve a remote over HTTP via the WebDAV protocol" ,
2017-10-02 21:29:23 +02:00
} , {
Value : "other" ,
Help : "Other site/service or software" ,
} } ,
} , {
2023-07-06 18:55:53 +02:00
Name : "user" ,
Help : "User name.\n\nIn case NTLM authentication is used, the username should be in the format 'Domain\\User'." ,
Sensitive : true ,
2017-10-02 21:29:23 +02:00
} , {
Name : "pass" ,
Help : "Password." ,
IsPassword : true ,
2018-06-25 17:03:36 +02:00
} , {
2023-07-06 18:55:53 +02:00
Name : "bearer_token" ,
Help : "Bearer token instead of user/pass (e.g. a Macaroon)." ,
Sensitive : true ,
2018-09-04 15:04:13 +02:00
} , {
Name : "bearer_token_command" ,
2021-08-16 11:30:01 +02:00
Help : "Command to run to get a bearer token." ,
2018-09-04 15:04:13 +02:00
Advanced : true ,
2021-02-21 16:53:06 +01:00
} , {
Name : config . ConfigEncoding ,
Help : configEncodingHelp ,
Advanced : true ,
2021-05-11 14:19:26 +02:00
} , {
Name : "headers" ,
2021-08-16 11:30:01 +02:00
Help : ` Set HTTP headers for all transactions .
2021-05-11 14:19:26 +02:00
Use this to set additional HTTP headers for all transactions
The input format is comma separated list of key , value pairs . Standard
[ CSV encoding ] ( https : //godoc.org/encoding/csv) may be used.
2021-11-04 12:50:43 +01:00
For example , to set a Cookie use ' Cookie , name = value ' , or ' "Cookie" , "name=value" ' .
2021-05-11 14:19:26 +02:00
You can set multiple headers , e . g . ' "Cookie" , "name=value" , "Authorization" , "xxx" ' .
` ,
Default : fs . CommaSepList { } ,
Advanced : true ,
2023-03-26 22:36:48 +02:00
} , {
Name : "pacer_min_sleep" ,
Help : "Minimum time to sleep between API calls." ,
Default : minSleep ,
Advanced : true ,
2022-05-20 11:06:55 +02:00
} , {
Name : "nextcloud_chunk_size" ,
Help : ` Nextcloud upload chunk size .
We recommend configuring your NextCloud instance to increase the max chunk size to 1 GB for better upload performances .
See https : //docs.nextcloud.com/server/latest/admin_manual/configuration_files/big_file_upload_configuration.html#adjust-chunk-size-on-nextcloud-side
Set to 0 to disable chunked uploading .
` ,
Advanced : true ,
Default : 10 * fs . Mebi , // Default NextCloud `max_chunk_size` is `10 MiB`. See https://github.com/nextcloud/server/blob/0447b53bda9fe95ea0cbed765aa332584605d652/apps/files/lib/App.php#L57
2017-10-02 21:29:23 +02:00
} } ,
} )
}
2018-05-14 19:06:57 +02:00
// Options defines the configuration for this backend
type Options struct {
2021-02-21 16:53:06 +01:00
URL string ` config:"url" `
Vendor string ` config:"vendor" `
User string ` config:"user" `
Pass string ` config:"pass" `
BearerToken string ` config:"bearer_token" `
BearerTokenCommand string ` config:"bearer_token_command" `
Enc encoder . MultiEncoder ` config:"encoding" `
2021-05-11 14:19:26 +02:00
Headers fs . CommaSepList ` config:"headers" `
2023-03-26 22:36:48 +02:00
PacerMinSleep fs . Duration ` config:"pacer_min_sleep" `
2022-05-20 11:06:55 +02:00
ChunkSize fs . SizeSuffix ` config:"nextcloud_chunk_size" `
2018-05-14 19:06:57 +02:00
}
2017-10-02 21:29:23 +02:00
// Fs represents a remote webdav
type Fs struct {
2018-08-04 12:02:47 +02:00
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
features * fs . Features // optional features
endpoint * url . URL // URL of the host
endpointURL string // endpoint as a string
2022-08-30 10:23:29 +02:00
srv * rest . Client // the connection to the server
2019-02-09 21:52:15 +01:00
pacer * fs . Pacer // pacer for API calls
2018-08-04 12:02:47 +02:00
precision time . Duration // mod time precision
canStream bool // set if can stream
useOCMtime bool // set if can use X-OC-Mtime
2023-04-28 18:38:49 +02:00
propsetMtime bool // set if can use propset
2018-08-04 12:02:47 +02:00
retryWithZeroDepth bool // some vendors (sharepoint) won't list files when Depth is 1 (our default)
2021-02-22 19:29:00 +01:00
checkBeforePurge bool // enables extra check that directory to purge really exists
2023-03-13 05:45:54 +01:00
hasOCMD5 bool // set if can use owncloud style checksums for MD5
hasOCSHA1 bool // set if can use owncloud style checksums for SHA1
hasMESHA1 bool // set if can use fastmail style checksums for SHA1
2021-02-22 02:07:05 +01:00
ntlmAuthMu sync . Mutex // mutex to serialize NTLM auth roundtrips
2022-05-20 11:06:55 +02:00
chunksUploadURL string // upload URL for nextcloud chunked
canChunk bool // set if nextcloud and nextcloud_chunk_size is set
2017-10-02 21:29:23 +02:00
}
// Object describes a webdav object
//
// Will definitely have info but maybe not meta
type Object struct {
fs * Fs // what this object is part of
remote string // The remote path
hasMetaData bool // whether info below has been set
size int64 // size of the object
modTime time . Time // modification time of the object
2019-01-27 14:33:21 +01:00
sha1 string // SHA-1 of the object content if known
md5 string // MD5 of the object content if known
2017-10-02 21:29:23 +02:00
}
// ------------------------------------------------------------
// Name of the remote (as passed into NewFs)
func ( f * Fs ) Name ( ) string {
return f . name
}
// Root of the remote (as passed into NewFs)
func ( f * Fs ) Root ( ) string {
return f . root
}
// String converts this Fs to a string
func ( f * Fs ) String ( ) string {
return fmt . Sprintf ( "webdav root '%s'" , f . root )
}
// Features returns the optional features of this Fs
func ( f * Fs ) Features ( ) * fs . Features {
return f . features
}
// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = [ ] int {
2019-06-15 11:58:13 +02:00
423 , // Locked
2017-10-02 21:29:23 +02:00
429 , // Too Many Requests.
500 , // Internal Server Error
502 , // Bad Gateway
503 , // Service Unavailable
504 , // Gateway Timeout
509 , // Bandwidth Limit Exceeded
}
// shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience
2021-03-11 15:44:01 +01:00
func ( f * Fs ) shouldRetry ( ctx context . Context , resp * http . Response , err error ) ( bool , error ) {
if fserrors . ContextError ( ctx , & err ) {
return false , err
}
2018-09-19 16:55:43 +02:00
// If we have a bearer token command and it has expired then refresh it
if f . opt . BearerTokenCommand != "" && resp != nil && resp . StatusCode == 401 {
fs . Debugf ( f , "Bearer token expired: %v" , err )
authErr := f . fetchAndSetBearerToken ( )
if authErr != nil {
err = authErr
}
return true , err
}
2018-01-12 17:30:54 +01:00
return fserrors . ShouldRetry ( err ) || fserrors . ShouldRetryHTTP ( resp , retryErrorCodes ) , err
2017-10-02 21:29:23 +02:00
}
2021-02-22 02:07:05 +01:00
// safeRoundTripper is a wrapper for http.RoundTripper that serializes
// http roundtrips. NTLM authentication sequence can involve up to four
// rounds of negotiations and might fail due to concurrency.
// This wrapper allows to use ntlmssp.Negotiator safely with goroutines.
type safeRoundTripper struct {
fs * Fs
rt http . RoundTripper
}
// RoundTrip guards wrapped RoundTripper by a mutex.
func ( srt * safeRoundTripper ) RoundTrip ( req * http . Request ) ( * http . Response , error ) {
srt . fs . ntlmAuthMu . Lock ( )
defer srt . fs . ntlmAuthMu . Unlock ( )
return srt . rt . RoundTrip ( req )
}
2017-10-25 13:00:23 +02:00
// itemIsDir returns true if the item is a directory
//
// When a client sees a resourcetype it doesn't recognize it should
// assume it is a regular non-collection resource. [WebDav book by
// Lisa Dusseault ch 7.5.8 p170]
func itemIsDir ( item * api . Response ) bool {
if t := item . Props . Type ; t != nil {
if t . Space == "DAV:" && t . Local == "collection" {
return true
}
fs . Debugf ( nil , "Unknown resource type %q/%q on %q" , t . Space , t . Local , item . Props . Name )
}
2018-11-19 14:12:24 +01:00
// the iscollection prop is a Microsoft extension, but if present it is a reliable indicator
2019-02-10 22:31:33 +01:00
// if the above check failed - see #2716. This can be an integer or a boolean - see #2964
2018-11-19 14:12:24 +01:00
if t := item . Props . IsCollection ; t != nil {
2019-02-10 22:31:33 +01:00
switch x := strings . ToLower ( * t ) ; x {
case "0" , "false" :
return false
case "1" , "true" :
return true
default :
fs . Debugf ( nil , "Unknown value %q for IsCollection" , x )
}
2018-11-19 14:12:24 +01:00
}
2017-10-25 13:00:23 +02:00
return false
}
2017-10-02 21:29:23 +02:00
// readMetaDataForPath reads the metadata from the path
2019-09-04 21:00:37 +02:00
func ( f * Fs ) readMetaDataForPath ( ctx context . Context , path string , depth string ) ( info * api . Prop , err error ) {
2017-10-02 21:29:23 +02:00
// FIXME how do we read back additional properties?
opts := rest . Opts {
Method : "PROPFIND" ,
Path : f . filePath ( path ) ,
2018-05-10 16:03:04 +02:00
ExtraHeaders : map [ string ] string {
2018-08-04 12:02:47 +02:00
"Depth" : depth ,
2018-05-10 16:03:04 +02:00
} ,
2018-06-18 13:22:13 +02:00
NoRedirect : true ,
2017-10-02 21:29:23 +02:00
}
2023-03-13 05:45:54 +01:00
if f . hasOCMD5 || f . hasOCSHA1 {
2019-01-27 14:33:21 +01:00
opts . Body = bytes . NewBuffer ( owncloudProps )
}
2017-10-02 21:29:23 +02:00
var result api . Multistatus
var resp * http . Response
err = f . pacer . Call ( func ( ) ( bool , error ) {
2019-09-04 21:00:37 +02:00
resp , err = f . srv . CallXML ( ctx , & opts , nil , & result )
2021-03-11 15:44:01 +01:00
return f . shouldRetry ( ctx , resp , err )
2017-10-02 21:29:23 +02:00
} )
if apiErr , ok := err . ( * api . Error ) ; ok {
// does not exist
2018-06-18 13:22:13 +02:00
switch apiErr . StatusCode {
case http . StatusNotFound :
2018-08-04 12:02:47 +02:00
if f . retryWithZeroDepth && depth != "0" {
2019-09-04 21:00:37 +02:00
return f . readMetaDataForPath ( ctx , path , "0" )
2018-08-04 12:02:47 +02:00
}
2018-06-18 13:22:13 +02:00
return nil , fs . ErrorObjectNotFound
case http . StatusMovedPermanently , http . StatusFound , http . StatusSeeOther :
// Some sort of redirect - go doesn't deal with these properly (it resets
// the method to GET). However we can assume that if it was redirected the
// object was not found.
2017-10-02 21:29:23 +02:00
return nil , fs . ErrorObjectNotFound
}
}
if err != nil {
2021-11-04 11:12:57 +01:00
return nil , fmt . Errorf ( "read metadata failed: %w" , err )
2017-10-02 21:29:23 +02:00
}
if len ( result . Responses ) < 1 {
return nil , fs . ErrorObjectNotFound
}
item := result . Responses [ 0 ]
if ! item . Props . StatusOK ( ) {
return nil , fs . ErrorObjectNotFound
}
2017-10-25 13:00:23 +02:00
if itemIsDir ( & item ) {
2021-09-06 14:54:08 +02:00
return nil , fs . ErrorIsDir
2017-10-02 21:29:23 +02:00
}
return & item . Props , nil
}
// errorHandler parses a non 2xx error response into an error
func errorHandler ( resp * http . Response ) error {
2018-05-10 16:02:41 +02:00
body , err := rest . ReadBody ( resp )
if err != nil {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "error when trying to read error from body: %w" , err )
2018-05-10 16:02:41 +02:00
}
2017-10-02 21:29:23 +02:00
// Decode error response
errResponse := new ( api . Error )
2018-05-10 16:02:41 +02:00
err = xml . Unmarshal ( body , & errResponse )
2017-10-02 21:29:23 +02:00
if err != nil {
2018-05-10 16:02:41 +02:00
// set the Message to be the body if can't parse the XML
errResponse . Message = strings . TrimSpace ( string ( body ) )
2017-10-02 21:29:23 +02:00
}
errResponse . Status = resp . Status
errResponse . StatusCode = resp . StatusCode
return errResponse
}
2019-02-07 18:41:17 +01:00
// addSlash makes sure s is terminated with a / if non empty
2017-10-02 21:29:23 +02:00
func addSlash ( s string ) string {
if s != "" && ! strings . HasSuffix ( s , "/" ) {
s += "/"
}
return s
}
// filePath returns a file path (f.root, file)
func ( f * Fs ) filePath ( file string ) string {
2021-02-21 16:53:06 +01:00
subPath := path . Join ( f . root , file )
if f . opt . Enc != encoder . EncodeZero {
subPath = f . opt . Enc . FromStandardPath ( subPath )
}
return rest . URLPathEscape ( subPath )
2017-10-02 21:29:23 +02:00
}
// dirPath returns a directory path (f.root, dir)
func ( f * Fs ) dirPath ( dir string ) string {
return addSlash ( f . filePath ( dir ) )
}
// filePath returns a file path (f.root, remote)
func ( o * Object ) filePath ( ) string {
return o . fs . filePath ( o . remote )
}
// NewFs constructs an Fs from the path, container:path
2020-11-05 16:18:51 +01:00
func NewFs ( ctx context . Context , name , root string , m configmap . Mapper ) ( fs . Fs , error ) {
2018-05-14 19:06:57 +02:00
// Parse config into Options struct
opt := new ( Options )
err := configstruct . Set ( m , opt )
if err != nil {
return nil , err
2017-10-02 21:29:23 +02:00
}
2021-05-11 14:19:26 +02:00
if len ( opt . Headers ) % 2 != 0 {
return nil , errors . New ( "odd number of headers supplied" )
}
fs . Debugf ( nil , "found headers: %v" , opt . Headers )
2018-06-18 13:13:47 +02:00
rootIsDir := strings . HasSuffix ( root , "/" )
2018-04-20 21:28:23 +02:00
root = strings . Trim ( root , "/" )
2017-10-02 21:29:23 +02:00
2018-05-14 19:06:57 +02:00
if ! strings . HasSuffix ( opt . URL , "/" ) {
opt . URL += "/"
}
if opt . Pass != "" {
2017-10-02 21:29:23 +02:00
var err error
2018-05-14 19:06:57 +02:00
opt . Pass , err = obscure . Reveal ( opt . Pass )
2017-10-02 21:29:23 +02:00
if err != nil {
2021-11-04 11:12:57 +01:00
return nil , fmt . Errorf ( "couldn't decrypt password: %w" , err )
2017-10-02 21:29:23 +02:00
}
}
2018-05-14 19:06:57 +02:00
if opt . Vendor == "" {
opt . Vendor = "other"
}
root = strings . Trim ( root , "/" )
2017-10-02 21:29:23 +02:00
2021-02-21 16:53:06 +01:00
if opt . Enc == encoder . EncodeZero && opt . Vendor == "sharepoint-ntlm" {
opt . Enc = defaultEncodingSharepointNTLM
}
2017-10-02 21:29:23 +02:00
// Parse the endpoint
2018-05-14 19:06:57 +02:00
u , err := url . Parse ( opt . URL )
2017-10-02 21:29:23 +02:00
if err != nil {
return nil , err
}
2021-02-22 02:07:05 +01:00
f := & Fs {
name : name ,
root : root ,
opt : * opt ,
endpoint : u ,
endpointURL : u . String ( ) ,
2023-03-26 22:36:48 +02:00
pacer : fs . NewPacer ( ctx , pacer . NewDefault ( pacer . MinSleep ( opt . PacerMinSleep ) , pacer . MaxSleep ( maxSleep ) , pacer . DecayConstant ( decayConstant ) ) ) ,
2021-02-22 02:07:05 +01:00
precision : fs . ModTimeNotSupported ,
}
2019-01-17 13:35:30 +01:00
client := fshttp . NewClient ( ctx )
if opt . Vendor == "sharepoint-ntlm" {
2021-02-06 15:50:53 +01:00
// Disable transparent HTTP/2 support as per https://golang.org/pkg/net/http/ ,
// otherwise any connection to IIS 10.0 fails with 'stream error: stream ID 39; HTTP_1_1_REQUIRED'
// https://docs.microsoft.com/en-us/iis/get-started/whats-new-in-iis-10/http2-on-iis says:
// 'Windows authentication (NTLM/Kerberos/Negotiate) is not supported with HTTP/2.'
t := fshttp . NewTransportCustom ( ctx , func ( t * http . Transport ) {
t . TLSNextProto = map [ string ] func ( string , * tls . Conn ) http . RoundTripper { }
} )
2021-02-22 02:07:05 +01:00
2019-01-17 13:35:30 +01:00
// Add NTLM layer
2021-02-22 02:07:05 +01:00
client . Transport = & safeRoundTripper {
fs : f ,
rt : ntlmssp . Negotiator { RoundTripper : t } ,
}
2017-10-02 21:29:23 +02:00
}
2021-02-22 02:07:05 +01:00
f . srv = rest . NewClient ( client ) . SetRoot ( u . String ( ) )
2017-10-02 21:29:23 +02:00
f . features = ( & fs . Features {
CanHaveEmptyDirectories : true ,
2020-11-05 17:00:40 +01:00
} ) . Fill ( ctx , f )
2018-11-17 14:14:54 +01:00
if opt . User != "" || opt . Pass != "" {
2018-05-14 19:06:57 +02:00
f . srv . SetUserPass ( opt . User , opt . Pass )
2018-11-17 14:14:54 +01:00
} else if opt . BearerToken != "" {
2018-09-04 15:04:13 +02:00
f . setBearerToken ( opt . BearerToken )
} else if f . opt . BearerTokenCommand != "" {
err = f . fetchAndSetBearerToken ( )
if err != nil {
return nil , err
}
2018-06-25 17:03:36 +02:00
}
2021-05-11 14:19:26 +02:00
if opt . Headers != nil {
f . addHeaders ( opt . Headers )
}
2017-10-02 21:29:23 +02:00
f . srv . SetErrorHandler ( errorHandler )
2019-09-04 21:21:10 +02:00
err = f . setQuirks ( ctx , opt . Vendor )
2018-04-09 10:05:43 +02:00
if err != nil {
return nil , err
}
2022-04-26 09:58:31 +02:00
if ! f . findHeader ( opt . Headers , "Referer" ) {
f . srv . SetHeader ( "Referer" , u . String ( ) )
}
2017-10-02 21:29:23 +02:00
2018-06-18 13:13:47 +02:00
if root != "" && ! rootIsDir {
2017-10-02 21:29:23 +02:00
// Check to see if the root actually an existing file
remote := path . Base ( root )
f . root = path . Dir ( root )
if f . root == "." {
f . root = ""
}
2019-06-17 10:34:30 +02:00
_ , err := f . NewObject ( ctx , remote )
2017-10-02 21:29:23 +02:00
if err != nil {
2021-11-04 11:12:57 +01:00
if errors . Is ( err , fs . ErrorObjectNotFound ) || errors . Is ( err , fs . ErrorIsDir ) {
2017-10-02 21:29:23 +02:00
// File doesn't exist so return old f
f . root = root
return f , nil
}
return nil , err
}
// return an error with an fs which points to the parent
return f , fs . ErrorIsFile
}
return f , nil
}
2018-09-04 15:04:13 +02:00
// sets the BearerToken up
func ( f * Fs ) setBearerToken ( token string ) {
f . opt . BearerToken = token
2019-11-21 12:06:20 +01:00
f . srv . SetHeader ( "Authorization" , "Bearer " + token )
2018-09-04 15:04:13 +02:00
}
// fetch the bearer token using the command
func ( f * Fs ) fetchBearerToken ( cmd string ) ( string , error ) {
var (
args = strings . Split ( cmd , " " )
stdout bytes . Buffer
stderr bytes . Buffer
c = exec . Command ( args [ 0 ] , args [ 1 : ] ... )
)
c . Stdout = & stdout
c . Stderr = & stderr
var (
err = c . Run ( )
stdoutString = strings . TrimSpace ( stdout . String ( ) )
stderrString = strings . TrimSpace ( stderr . String ( ) )
)
if err != nil {
if stderrString == "" {
stderrString = stdoutString
}
2021-11-04 11:12:57 +01:00
return "" , fmt . Errorf ( "failed to get bearer token using %q: %s: %w" , f . opt . BearerTokenCommand , stderrString , err )
2018-09-04 15:04:13 +02:00
}
return stdoutString , nil
}
2021-05-11 14:19:26 +02:00
// Adds the configured headers to the request if any
func ( f * Fs ) addHeaders ( headers fs . CommaSepList ) {
for i := 0 ; i < len ( headers ) ; i += 2 {
key := f . opt . Headers [ i ]
value := f . opt . Headers [ i + 1 ]
f . srv . SetHeader ( key , value )
}
}
2022-04-26 09:58:31 +02:00
// Returns true if the header was configured
func ( f * Fs ) findHeader ( headers fs . CommaSepList , find string ) bool {
for i := 0 ; i < len ( headers ) ; i += 2 {
key := f . opt . Headers [ i ]
if strings . EqualFold ( key , find ) {
return true
}
}
return false
}
2018-09-04 15:04:13 +02:00
// fetch the bearer token and set it if successful
func ( f * Fs ) fetchAndSetBearerToken ( ) error {
if f . opt . BearerTokenCommand == "" {
return nil
}
token , err := f . fetchBearerToken ( f . opt . BearerTokenCommand )
if err != nil {
return err
}
f . setBearerToken ( token )
return nil
}
2023-07-02 16:50:52 +02:00
// The WebDAV url can optionally be suffixed with a path. This suffix needs to be ignored for determining the temporary upload directory of chunks.
var nextCloudURLRegex = regexp . MustCompile ( ` ^(.*)/dav/files/([^/]+) ` )
2022-05-20 11:06:55 +02:00
2017-10-02 21:29:23 +02:00
// setQuirks adjusts the Fs for the vendor passed in
2019-09-04 21:21:10 +02:00
func ( f * Fs ) setQuirks ( ctx context . Context , vendor string ) error {
2017-10-02 21:29:23 +02:00
switch vendor {
2023-03-13 05:45:54 +01:00
case "fastmail" :
f . canStream = true
f . precision = time . Second
f . useOCMtime = true
f . hasMESHA1 = true
2017-10-02 21:29:23 +02:00
case "owncloud" :
f . canStream = true
f . precision = time . Second
f . useOCMtime = true
2023-04-28 18:38:49 +02:00
f . propsetMtime = true
2023-03-13 05:45:54 +01:00
f . hasOCMD5 = true
f . hasOCSHA1 = true
2017-10-02 21:29:23 +02:00
case "nextcloud" :
f . precision = time . Second
f . useOCMtime = true
2023-04-28 18:38:49 +02:00
f . propsetMtime = true
2023-03-13 05:45:54 +01:00
f . hasOCSHA1 = true
2022-05-20 11:06:55 +02:00
f . canChunk = true
2023-07-02 16:50:52 +02:00
if f . opt . ChunkSize == 0 {
fs . Logf ( nil , "Chunked uploads are disabled because nextcloud_chunk_size is set to 0" )
} else {
chunksUploadURL , err := f . getChunksUploadURL ( )
if err != nil {
return err
}
f . chunksUploadURL = chunksUploadURL
fs . Logf ( nil , "Chunks temporary upload directory: %s" , f . chunksUploadURL )
2022-05-20 11:06:55 +02:00
}
2018-04-09 10:05:43 +02:00
case "sharepoint" :
// To mount sharepoint, two Cookies are required
// They have to be set instead of BasicAuth
f . srv . RemoveHeader ( "Authorization" ) // We don't need this Header if using cookies
2018-05-14 19:06:57 +02:00
spCk := odrvcookie . New ( f . opt . User , f . opt . Pass , f . endpointURL )
2019-09-04 21:21:10 +02:00
spCookies , err := spCk . Cookies ( ctx )
2018-04-09 10:05:43 +02:00
if err != nil {
return err
}
2018-09-17 22:49:02 +02:00
odrvcookie . NewRenew ( 12 * time . Hour , func ( ) {
2019-09-04 21:21:10 +02:00
spCookies , err := spCk . Cookies ( ctx )
2018-09-17 22:49:02 +02:00
if err != nil {
fs . Errorf ( "could not renew cookies: %s" , err . Error ( ) )
return
}
f . srv . SetCookie ( & spCookies . FedAuth , & spCookies . RtFa )
fs . Debugf ( spCookies , "successfully renewed sharepoint cookies" )
} )
2018-04-09 10:05:43 +02:00
f . srv . SetCookie ( & spCookies . FedAuth , & spCookies . RtFa )
2018-08-04 12:02:47 +02:00
// sharepoint, unlike the other vendors, only lists files if the depth header is set to 0
// however, rclone defaults to 1 since it provides recursive directory listing
// to determine if we may have found a file, the request has to be resent
// with the depth set to 0
f . retryWithZeroDepth = true
2019-01-17 13:35:30 +01:00
case "sharepoint-ntlm" :
// Sharepoint with NTLM authentication
// See comment above
f . retryWithZeroDepth = true
2021-02-22 19:29:00 +01:00
// Sharepoint 2016 returns status 204 to the purge request
// even if the directory to purge does not really exist
// so we must perform an extra check to detect this
// condition and return a proper error code.
f . checkBeforePurge = true
2023-11-05 13:37:25 +01:00
case "rclone" :
f . canStream = true
f . precision = time . Second
f . useOCMtime = true
2017-10-02 21:29:23 +02:00
case "other" :
default :
fs . Debugf ( f , "Unknown vendor %q" , vendor )
}
// Remove PutStream from optional features
if ! f . canStream {
f . features . PutStream = nil
}
2018-04-09 10:05:43 +02:00
return nil
2017-10-02 21:29:23 +02:00
}
// Return an Object from a path
//
// If it can't be found it returns the error fs.ErrorObjectNotFound.
2019-09-04 21:00:37 +02:00
func ( f * Fs ) newObjectWithInfo ( ctx context . Context , remote string , info * api . Prop ) ( fs . Object , error ) {
2017-10-02 21:29:23 +02:00
o := & Object {
fs : f ,
remote : remote ,
}
var err error
if info != nil {
// Set info
err = o . setMetaData ( info )
} else {
2019-09-04 21:00:37 +02:00
err = o . readMetaData ( ctx ) // reads info and meta, returning an error
2017-10-02 21:29:23 +02:00
}
if err != nil {
return nil , err
}
return o , nil
}
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
2019-06-17 10:34:30 +02:00
func ( f * Fs ) NewObject ( ctx context . Context , remote string ) ( fs . Object , error ) {
2019-09-04 21:00:37 +02:00
return f . newObjectWithInfo ( ctx , remote , nil )
2017-10-02 21:29:23 +02:00
}
2019-01-27 14:33:21 +01:00
// Read the normal props, plus the checksums
//
// <oc:checksums><oc:checksum>SHA1:f572d396fae9206628714fb2ce00f72e94f2258f MD5:b1946ac92492d2347c6235b4d2611184 ADLER32:084b021f</oc:checksum></oc:checksums>
var owncloudProps = [ ] byte ( ` < ? xml version = "1.0" ? >
< d : propfind xmlns : d = "DAV:" xmlns : oc = "http://owncloud.org/ns" xmlns : nc = "http://nextcloud.org/ns" >
< d : prop >
< d : displayname / >
< d : getlastmodified / >
< d : getcontentlength / >
< d : resourcetype / >
< d : getcontenttype / >
< oc : checksums / >
< / d : prop >
< / d : propfind >
` )
2017-10-02 21:29:23 +02:00
// list the objects into the function supplied
//
// If directories is set it only sends directories
// User function to process a File item from listAll
//
// Should return true to finish processing
type listAllFn func ( string , bool , * api . Prop ) bool
// Lists the directory required calling the user function on each item found
//
// If the user fn ever returns true then it early exits with found = true
2019-09-04 21:00:37 +02:00
func ( f * Fs ) listAll ( ctx context . Context , dir string , directoriesOnly bool , filesOnly bool , depth string , fn listAllFn ) ( found bool , err error ) {
2017-10-02 21:29:23 +02:00
opts := rest . Opts {
Method : "PROPFIND" ,
Path : f . dirPath ( dir ) , // FIXME Should not start with /
2017-10-23 13:13:01 +02:00
ExtraHeaders : map [ string ] string {
2018-08-04 12:02:47 +02:00
"Depth" : depth ,
2017-10-23 13:13:01 +02:00
} ,
2017-10-02 21:29:23 +02:00
}
2023-03-13 05:45:54 +01:00
if f . hasOCMD5 || f . hasOCSHA1 {
2019-01-27 14:33:21 +01:00
opts . Body = bytes . NewBuffer ( owncloudProps )
}
2017-10-02 21:29:23 +02:00
var result api . Multistatus
var resp * http . Response
err = f . pacer . Call ( func ( ) ( bool , error ) {
2019-09-04 21:00:37 +02:00
resp , err = f . srv . CallXML ( ctx , & opts , nil , & result )
2021-03-11 15:44:01 +01:00
return f . shouldRetry ( ctx , resp , err )
2017-10-02 21:29:23 +02:00
} )
if err != nil {
if apiErr , ok := err . ( * api . Error ) ; ok {
// does not exist
if apiErr . StatusCode == http . StatusNotFound {
2018-08-04 12:02:47 +02:00
if f . retryWithZeroDepth && depth != "0" {
2019-09-04 21:00:37 +02:00
return f . listAll ( ctx , dir , directoriesOnly , filesOnly , "0" , fn )
2018-08-04 12:02:47 +02:00
}
2017-10-02 21:29:23 +02:00
return found , fs . ErrorDirNotFound
}
}
2021-11-04 11:12:57 +01:00
return found , fmt . Errorf ( "couldn't list files: %w" , err )
2017-10-02 21:29:23 +02:00
}
//fmt.Printf("result = %#v", &result)
baseURL , err := rest . URLJoin ( f . endpoint , opts . Path )
if err != nil {
2021-11-04 11:12:57 +01:00
return false , fmt . Errorf ( "couldn't join URL: %w" , err )
2017-10-02 21:29:23 +02:00
}
for i := range result . Responses {
item := & result . Responses [ i ]
2017-10-25 13:00:23 +02:00
isDir := itemIsDir ( item )
2017-10-02 21:29:23 +02:00
// Find name
u , err := rest . URLJoin ( baseURL , item . Href )
if err != nil {
fs . Errorf ( nil , "URL Join failed for %q and %q: %v" , baseURL , item . Href , err )
continue
}
2017-10-23 13:13:01 +02:00
// Make sure directories end with a /
if isDir {
u . Path = addSlash ( u . Path )
}
2017-10-02 21:29:23 +02:00
if ! strings . HasPrefix ( u . Path , baseURL . Path ) {
2017-10-23 13:13:01 +02:00
fs . Debugf ( nil , "Item with unknown path received: %q, %q" , u . Path , baseURL . Path )
2017-10-02 21:29:23 +02:00
continue
}
2021-02-21 16:53:06 +01:00
subPath := u . Path [ len ( baseURL . Path ) : ]
2023-01-31 13:07:21 +01:00
subPath = strings . TrimPrefix ( subPath , "/" ) // ignore leading / here for davrods
2021-02-21 16:53:06 +01:00
if f . opt . Enc != encoder . EncodeZero {
subPath = f . opt . Enc . ToStandardPath ( subPath )
}
remote := path . Join ( dir , subPath )
2022-06-08 22:25:17 +02:00
remote = strings . TrimSuffix ( remote , "/" )
2017-10-02 21:29:23 +02:00
// the listing contains info about itself which we ignore
if remote == dir {
continue
}
// Check OK
if ! item . Props . StatusOK ( ) {
fs . Debugf ( remote , "Ignoring item with bad status %q" , item . Props . Status )
continue
}
if isDir {
if filesOnly {
continue
}
} else {
if directoriesOnly {
continue
}
}
// item.Name = restoreReservedChars(item.Name)
if fn ( remote , isDir , & item . Props ) {
found = true
break
}
}
return
}
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
2019-06-17 10:34:30 +02:00
func ( f * Fs ) List ( ctx context . Context , dir string ) ( entries fs . DirEntries , err error ) {
2017-10-02 21:29:23 +02:00
var iErr error
2019-09-04 21:00:37 +02:00
_ , err = f . listAll ( ctx , dir , false , false , defaultDepth , func ( remote string , isDir bool , info * api . Prop ) bool {
2017-10-02 21:29:23 +02:00
if isDir {
d := fs . NewDir ( remote , time . Time ( info . Modified ) )
// .SetID(info.ID)
// FIXME more info from dir? can set size, items?
entries = append ( entries , d )
} else {
2019-09-04 21:00:37 +02:00
o , err := f . newObjectWithInfo ( ctx , remote , info )
2017-10-02 21:29:23 +02:00
if err != nil {
iErr = err
return true
}
entries = append ( entries , o )
}
return false
} )
if err != nil {
return nil , err
}
if iErr != nil {
return nil , iErr
}
return entries , nil
}
// Creates from the parameters passed in a half finished Object which
// must have setMetaData called on it
//
// Used to create new objects
func ( f * Fs ) createObject ( remote string , modTime time . Time , size int64 ) ( o * Object ) {
// Temporary Object under construction
o = & Object {
fs : f ,
remote : remote ,
size : size ,
modTime : modTime ,
}
return o
}
// Put the object
//
2022-08-05 17:35:41 +02:00
// Copy the reader in to the new object which is returned.
2017-10-02 21:29:23 +02:00
//
// The new object may have been created if an error is returned
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Put ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( fs . Object , error ) {
o := f . createObject ( src . Remote ( ) , src . ModTime ( ctx ) , src . Size ( ) )
return o , o . Update ( ctx , in , src , options ... )
2017-10-02 21:29:23 +02:00
}
// PutStream uploads to the remote path with the modTime given of indeterminate size
2019-06-17 10:34:30 +02:00
func ( f * Fs ) PutStream ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( fs . Object , error ) {
return f . Put ( ctx , in , src , options ... )
2017-10-02 21:29:23 +02:00
}
// mkParentDir makes the parent of the native path dirPath if
// necessary and any directories above that
2020-07-17 17:48:31 +02:00
func ( f * Fs ) mkParentDir ( ctx context . Context , dirPath string ) ( err error ) {
// defer log.Trace(dirPath, "")("err=%v", &err)
2018-06-14 10:49:01 +02:00
// chop off trailing / if it exists
2022-06-08 22:25:17 +02:00
parent := path . Dir ( strings . TrimSuffix ( dirPath , "/" ) )
2017-10-02 21:29:23 +02:00
if parent == "." {
parent = ""
}
2019-06-17 10:34:30 +02:00
return f . mkdir ( ctx , parent )
2017-10-02 21:29:23 +02:00
}
2020-07-17 17:48:31 +02:00
// _dirExists - list dirPath to see if it exists
//
// dirPath should be a native path ending in a /
func ( f * Fs ) _dirExists ( ctx context . Context , dirPath string ) ( exists bool ) {
opts := rest . Opts {
Method : "PROPFIND" ,
Path : dirPath ,
ExtraHeaders : map [ string ] string {
"Depth" : "0" ,
} ,
}
var result api . Multistatus
var resp * http . Response
var err error
err = f . pacer . Call ( func ( ) ( bool , error ) {
resp , err = f . srv . CallXML ( ctx , & opts , nil , & result )
2021-03-11 15:44:01 +01:00
return f . shouldRetry ( ctx , resp , err )
2020-07-17 17:48:31 +02:00
} )
return err == nil
}
2018-11-29 18:54:02 +01:00
// low level mkdir, only makes the directory, doesn't attempt to create parents
2019-09-04 21:00:37 +02:00
func ( f * Fs ) _mkdir ( ctx context . Context , dirPath string ) error {
2018-11-29 18:54:02 +01:00
// We assume the root is already created
2017-10-02 21:29:23 +02:00
if dirPath == "" {
return nil
}
2018-06-14 10:49:01 +02:00
// Collections must end with /
if ! strings . HasSuffix ( dirPath , "/" ) {
dirPath += "/"
}
2017-10-02 21:29:23 +02:00
opts := rest . Opts {
Method : "MKCOL" ,
Path : dirPath ,
NoResponse : true ,
}
2019-03-08 13:54:39 +01:00
err := f . pacer . Call ( func ( ) ( bool , error ) {
2019-09-04 21:00:37 +02:00
resp , err := f . srv . Call ( ctx , & opts )
2021-03-11 15:44:01 +01:00
return f . shouldRetry ( ctx , resp , err )
2017-10-02 21:29:23 +02:00
} )
2019-03-08 13:54:39 +01:00
if apiErr , ok := err . ( * api . Error ) ; ok {
2020-07-17 17:48:31 +02:00
// Check if it already exists. The response code for this isn't
// defined in the RFC so the implementations vary wildly.
//
2019-03-08 13:54:39 +01:00
// owncloud returns 423/StatusLocked if the create is already in progress
if apiErr . StatusCode == http . StatusMethodNotAllowed || apiErr . StatusCode == http . StatusNotAcceptable || apiErr . StatusCode == http . StatusLocked {
return nil
}
2020-07-17 17:48:31 +02:00
// 4shared returns a 409/StatusConflict here which clashes
// horribly with the intermediate paths don't exist meaning. So
// check to see if actually exists. This will correct other
// error codes too.
if f . _dirExists ( ctx , dirPath ) {
return nil
}
2019-03-08 13:54:39 +01:00
}
return err
2018-11-29 18:54:02 +01:00
}
// mkdir makes the directory and parents using native paths
2020-07-17 17:48:31 +02:00
func ( f * Fs ) mkdir ( ctx context . Context , dirPath string ) ( err error ) {
// defer log.Trace(dirPath, "")("err=%v", &err)
err = f . _mkdir ( ctx , dirPath )
2017-10-02 21:29:23 +02:00
if apiErr , ok := err . ( * api . Error ) ; ok {
2019-03-08 13:54:39 +01:00
// parent does not exist so create it first then try again
2017-10-02 21:29:23 +02:00
if apiErr . StatusCode == http . StatusConflict {
2019-06-17 10:34:30 +02:00
err = f . mkParentDir ( ctx , dirPath )
2017-10-02 21:29:23 +02:00
if err == nil {
2019-09-04 21:00:37 +02:00
err = f . _mkdir ( ctx , dirPath )
2017-10-02 21:29:23 +02:00
}
}
}
return err
}
// Mkdir creates the directory if it doesn't exist
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Mkdir ( ctx context . Context , dir string ) error {
2017-10-02 21:29:23 +02:00
dirPath := f . dirPath ( dir )
2019-06-17 10:34:30 +02:00
return f . mkdir ( ctx , dirPath )
2017-10-02 21:29:23 +02:00
}
// dirNotEmpty returns true if the directory exists and is not Empty
//
// if the directory does not exist then err will be ErrorDirNotFound
2019-09-04 21:00:37 +02:00
func ( f * Fs ) dirNotEmpty ( ctx context . Context , dir string ) ( found bool , err error ) {
return f . listAll ( ctx , dir , false , false , defaultDepth , func ( remote string , isDir bool , info * api . Prop ) bool {
2017-10-02 21:29:23 +02:00
return true
} )
}
// purgeCheck removes the root directory, if check is set then it
// refuses to do so if it has anything in
2019-09-04 21:00:37 +02:00
func ( f * Fs ) purgeCheck ( ctx context . Context , dir string , check bool ) error {
2017-10-02 21:29:23 +02:00
if check {
2019-09-04 21:00:37 +02:00
notEmpty , err := f . dirNotEmpty ( ctx , dir )
2017-10-02 21:29:23 +02:00
if err != nil {
return err
}
if notEmpty {
return fs . ErrorDirectoryNotEmpty
}
2021-02-22 19:29:00 +01:00
} else if f . checkBeforePurge {
// We are doing purge as the `check` argument is unset.
// The quirk says that we are working with Sharepoint 2016.
// This provider returns status 204 even if the purged directory
// does not really exist so we perform an extra check here.
// Only the existence is checked, all other errors must be
// ignored here to make the rclone test suite pass.
depth := defaultDepth
if f . retryWithZeroDepth {
depth = "0"
}
_ , err := f . readMetaDataForPath ( ctx , dir , depth )
if err == fs . ErrorObjectNotFound {
return fs . ErrorDirNotFound
}
2017-10-02 21:29:23 +02:00
}
opts := rest . Opts {
Method : "DELETE" ,
Path : f . dirPath ( dir ) ,
NoResponse : true ,
}
var resp * http . Response
var err error
err = f . pacer . Call ( func ( ) ( bool , error ) {
2019-09-04 21:00:37 +02:00
resp , err = f . srv . CallXML ( ctx , & opts , nil , nil )
2021-03-11 15:44:01 +01:00
return f . shouldRetry ( ctx , resp , err )
2017-10-02 21:29:23 +02:00
} )
if err != nil {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "rmdir failed: %w" , err )
2017-10-02 21:29:23 +02:00
}
// FIXME parse Multistatus response
return nil
}
// Rmdir deletes the root folder
//
// Returns an error if it isn't empty
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Rmdir ( ctx context . Context , dir string ) error {
2019-09-04 21:00:37 +02:00
return f . purgeCheck ( ctx , dir , true )
2017-10-02 21:29:23 +02:00
}
// Precision return the precision of this Fs
func ( f * Fs ) Precision ( ) time . Duration {
return f . precision
}
2020-10-13 23:43:40 +02:00
// Copy or Move src to this remote using server-side copy operations.
2017-10-02 21:29:23 +02:00
//
2022-08-05 17:35:41 +02:00
// This is stored with the remote path given.
2017-10-02 21:29:23 +02:00
//
2022-08-05 17:35:41 +02:00
// It returns the destination Object and a possible error.
2017-10-02 21:29:23 +02:00
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy/fs.ErrorCantMove
2019-06-17 10:34:30 +02:00
func ( f * Fs ) copyOrMove ( ctx context . Context , src fs . Object , remote string , method string ) ( fs . Object , error ) {
2017-10-02 21:29:23 +02:00
srcObj , ok := src . ( * Object )
if ! ok {
fs . Debugf ( src , "Can't copy - not same remote type" )
if method == "COPY" {
return nil , fs . ErrorCantCopy
}
return nil , fs . ErrorCantMove
}
2022-11-15 10:51:30 +01:00
srcFs := srcObj . fs
2017-10-02 21:29:23 +02:00
dstPath := f . filePath ( remote )
2019-06-17 10:34:30 +02:00
err := f . mkParentDir ( ctx , dstPath )
2017-10-02 21:29:23 +02:00
if err != nil {
2022-05-20 11:06:55 +02:00
return nil , fmt . Errorf ( "copy mkParentDir failed: %w" , err )
2017-10-02 21:29:23 +02:00
}
2017-10-25 23:56:47 +02:00
destinationURL , err := rest . URLJoin ( f . endpoint , dstPath )
if err != nil {
2021-11-04 11:12:57 +01:00
return nil , fmt . Errorf ( "copyOrMove couldn't join URL: %w" , err )
2017-10-25 23:56:47 +02:00
}
2017-10-02 21:29:23 +02:00
var resp * http . Response
opts := rest . Opts {
Method : method ,
Path : srcObj . filePath ( ) ,
NoResponse : true ,
ExtraHeaders : map [ string ] string {
2017-10-25 23:56:47 +02:00
"Destination" : destinationURL . String ( ) ,
2023-04-24 15:35:42 +02:00
"Overwrite" : "T" ,
2017-10-02 21:29:23 +02:00
} ,
}
if f . useOCMtime {
2020-02-06 12:49:49 +01:00
opts . ExtraHeaders [ "X-OC-Mtime" ] = fmt . Sprintf ( "%d" , src . ModTime ( ctx ) . Unix ( ) )
2017-10-02 21:29:23 +02:00
}
2022-11-15 10:51:30 +01:00
// Direct the MOVE/COPY to the source server
err = srcFs . pacer . Call ( func ( ) ( bool , error ) {
resp , err = srcFs . srv . Call ( ctx , & opts )
return srcFs . shouldRetry ( ctx , resp , err )
2017-10-02 21:29:23 +02:00
} )
if err != nil {
2022-05-20 11:06:55 +02:00
return nil , fmt . Errorf ( "copy call failed: %w" , err )
2017-10-02 21:29:23 +02:00
}
2019-06-17 10:34:30 +02:00
dstObj , err := f . NewObject ( ctx , remote )
2017-10-02 21:29:23 +02:00
if err != nil {
2022-05-20 11:06:55 +02:00
return nil , fmt . Errorf ( "copy NewObject failed: %w" , err )
2017-10-02 21:29:23 +02:00
}
2023-06-26 18:52:31 +02:00
if f . useOCMtime && resp . Header . Get ( "X-OC-Mtime" ) != "accepted" && f . propsetMtime {
fs . Debugf ( dstObj , "Setting modtime after copy to %v" , src . ModTime ( ctx ) )
err = dstObj . SetModTime ( ctx , src . ModTime ( ctx ) )
if err != nil {
return nil , fmt . Errorf ( "failed to set modtime: %w" , err )
}
}
2017-10-02 21:29:23 +02:00
return dstObj , nil
}
2020-10-13 23:43:40 +02:00
// Copy src to this remote using server-side copy operations.
2017-10-02 21:29:23 +02:00
//
2022-08-05 17:35:41 +02:00
// This is stored with the remote path given.
2017-10-02 21:29:23 +02:00
//
2022-08-05 17:35:41 +02:00
// It returns the destination Object and a possible error.
2017-10-02 21:29:23 +02:00
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Copy ( ctx context . Context , src fs . Object , remote string ) ( fs . Object , error ) {
return f . copyOrMove ( ctx , src , remote , "COPY" )
2017-10-02 21:29:23 +02:00
}
2020-06-04 23:25:14 +02:00
// Purge deletes all the files in the directory
2017-10-02 21:29:23 +02:00
//
// Optional interface: Only implement this if you have a way of
// deleting all the files quicker than just running Remove() on the
// result of List()
2020-06-04 23:25:14 +02:00
func ( f * Fs ) Purge ( ctx context . Context , dir string ) error {
return f . purgeCheck ( ctx , dir , false )
2017-10-02 21:29:23 +02:00
}
2020-10-13 23:43:40 +02:00
// Move src to this remote using server-side move operations.
2017-10-02 21:29:23 +02:00
//
2022-08-05 17:35:41 +02:00
// This is stored with the remote path given.
2017-10-02 21:29:23 +02:00
//
2022-08-05 17:35:41 +02:00
// It returns the destination Object and a possible error.
2017-10-02 21:29:23 +02:00
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
2019-06-17 10:34:30 +02:00
func ( f * Fs ) Move ( ctx context . Context , src fs . Object , remote string ) ( fs . Object , error ) {
return f . copyOrMove ( ctx , src , remote , "MOVE" )
2017-10-02 21:29:23 +02:00
}
// DirMove moves src, srcRemote to this remote at dstRemote
2020-10-13 23:43:40 +02:00
// using server-side move operations.
2017-10-02 21:29:23 +02:00
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
2019-06-17 10:34:30 +02:00
func ( f * Fs ) DirMove ( ctx context . Context , src fs . Fs , srcRemote , dstRemote string ) error {
2017-10-02 21:29:23 +02:00
srcFs , ok := src . ( * Fs )
if ! ok {
fs . Debugf ( srcFs , "Can't move directory - not same remote type" )
return fs . ErrorCantDirMove
}
srcPath := srcFs . filePath ( srcRemote )
dstPath := f . filePath ( dstRemote )
// Check if destination exists
2019-09-04 21:00:37 +02:00
_ , err := f . dirNotEmpty ( ctx , dstRemote )
2017-10-02 21:29:23 +02:00
if err == nil {
return fs . ErrorDirExists
}
if err != fs . ErrorDirNotFound {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "DirMove dirExists dst failed: %w" , err )
2017-10-02 21:29:23 +02:00
}
// Make sure the parent directory exists
2019-06-17 10:34:30 +02:00
err = f . mkParentDir ( ctx , dstPath )
2017-10-02 21:29:23 +02:00
if err != nil {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "DirMove mkParentDir dst failed: %w" , err )
2017-10-02 21:29:23 +02:00
}
2017-10-25 23:56:47 +02:00
destinationURL , err := rest . URLJoin ( f . endpoint , dstPath )
if err != nil {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "DirMove couldn't join URL: %w" , err )
2017-10-25 23:56:47 +02:00
}
2017-10-02 21:29:23 +02:00
var resp * http . Response
opts := rest . Opts {
Method : "MOVE" ,
Path : addSlash ( srcPath ) ,
NoResponse : true ,
ExtraHeaders : map [ string ] string {
2017-10-25 23:56:47 +02:00
"Destination" : addSlash ( destinationURL . String ( ) ) ,
2023-04-24 15:35:42 +02:00
"Overwrite" : "T" ,
2017-10-02 21:29:23 +02:00
} ,
}
2022-11-15 10:51:30 +01:00
// Direct the MOVE/COPY to the source server
err = srcFs . pacer . Call ( func ( ) ( bool , error ) {
resp , err = srcFs . srv . Call ( ctx , & opts )
return srcFs . shouldRetry ( ctx , resp , err )
2017-10-02 21:29:23 +02:00
} )
if err != nil {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "DirMove MOVE call failed: %w" , err )
2017-10-02 21:29:23 +02:00
}
return nil
}
// Hashes returns the supported hash sets.
2018-01-12 17:30:54 +01:00
func ( f * Fs ) Hashes ( ) hash . Set {
2019-11-24 14:08:06 +01:00
hashes := hash . Set ( hash . None )
2023-03-13 05:45:54 +01:00
if f . hasOCMD5 {
2019-11-24 14:08:06 +01:00
hashes . Add ( hash . MD5 )
2019-01-27 14:33:21 +01:00
}
2023-03-13 05:45:54 +01:00
if f . hasOCSHA1 || f . hasMESHA1 {
2019-11-24 14:08:06 +01:00
hashes . Add ( hash . SHA1 )
}
return hashes
2017-10-02 21:29:23 +02:00
}
2019-01-27 17:01:42 +01:00
// About gets quota information
2019-06-17 10:34:30 +02:00
func ( f * Fs ) About ( ctx context . Context ) ( * fs . Usage , error ) {
2019-01-27 17:01:42 +01:00
opts := rest . Opts {
Method : "PROPFIND" ,
Path : "" ,
ExtraHeaders : map [ string ] string {
"Depth" : "0" ,
} ,
}
opts . Body = bytes . NewBuffer ( [ ] byte ( ` < ? xml version = "1.0" ? >
< D : propfind xmlns : D = "DAV:" >
< D : prop >
< D : quota - available - bytes / >
< D : quota - used - bytes / >
< / D : prop >
< / D : propfind >
` ) )
2020-06-14 13:11:19 +02:00
var q api . Quota
2019-01-27 17:01:42 +01:00
var resp * http . Response
var err error
err = f . pacer . Call ( func ( ) ( bool , error ) {
2019-09-04 21:00:37 +02:00
resp , err = f . srv . CallXML ( ctx , & opts , nil , & q )
2021-03-11 15:44:01 +01:00
return f . shouldRetry ( ctx , resp , err )
2019-01-27 17:01:42 +01:00
} )
if err != nil {
2022-01-14 22:18:32 +01:00
return nil , err
2019-01-27 17:01:42 +01:00
}
usage := & fs . Usage { }
2020-06-14 13:11:19 +02:00
if i , err := strconv . ParseInt ( q . Used , 10 , 64 ) ; err == nil && i >= 0 {
usage . Used = fs . NewUsageValue ( i )
2020-03-01 14:41:48 +01:00
}
2020-06-14 13:11:19 +02:00
if i , err := strconv . ParseInt ( q . Available , 10 , 64 ) ; err == nil && i >= 0 {
usage . Free = fs . NewUsageValue ( i )
2020-03-01 14:41:48 +01:00
}
2020-06-14 13:11:19 +02:00
if usage . Used != nil && usage . Free != nil {
usage . Total = fs . NewUsageValue ( * usage . Used + * usage . Free )
2019-01-27 17:01:42 +01:00
}
return usage , nil
}
2017-10-02 21:29:23 +02:00
// ------------------------------------------------------------
// Fs returns the parent Fs
func ( o * Object ) Fs ( ) fs . Info {
return o . fs
}
// Return a string version
func ( o * Object ) String ( ) string {
if o == nil {
return "<nil>"
}
return o . remote
}
// Remote returns the remote path
func ( o * Object ) Remote ( ) string {
return o . remote
}
2019-01-27 14:33:21 +01:00
// Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
2019-06-17 10:34:30 +02:00
func ( o * Object ) Hash ( ctx context . Context , t hash . Type ) ( string , error ) {
2023-03-13 05:45:54 +01:00
if t == hash . MD5 && o . fs . hasOCMD5 {
2019-11-24 14:08:06 +01:00
return o . md5 , nil
}
2023-03-13 05:45:54 +01:00
if t == hash . SHA1 && ( o . fs . hasOCSHA1 || o . fs . hasMESHA1 ) {
2019-11-24 14:08:06 +01:00
return o . sha1 , nil
2017-10-02 21:29:23 +02:00
}
2019-01-27 14:33:21 +01:00
return "" , hash . ErrUnsupported
2017-10-02 21:29:23 +02:00
}
// Size returns the size of an object in bytes
func ( o * Object ) Size ( ) int64 {
2019-09-04 21:00:37 +02:00
ctx := context . TODO ( )
err := o . readMetaData ( ctx )
2017-10-02 21:29:23 +02:00
if err != nil {
fs . Logf ( o , "Failed to read metadata: %v" , err )
return 0
}
return o . size
}
// setMetaData sets the metadata from info
func ( o * Object ) setMetaData ( info * api . Prop ) ( err error ) {
o . hasMetaData = true
o . size = info . Size
o . modTime = time . Time ( info . Modified )
2023-03-13 05:45:54 +01:00
if o . fs . hasOCMD5 || o . fs . hasOCSHA1 || o . fs . hasMESHA1 {
2019-01-27 14:33:21 +01:00
hashes := info . Hashes ( )
2023-03-13 05:45:54 +01:00
if o . fs . hasOCSHA1 || o . fs . hasMESHA1 {
2019-11-24 14:08:06 +01:00
o . sha1 = hashes [ hash . SHA1 ]
}
2023-03-13 05:45:54 +01:00
if o . fs . hasOCMD5 {
2019-11-24 14:08:06 +01:00
o . md5 = hashes [ hash . MD5 ]
}
2019-01-27 14:33:21 +01:00
}
2017-10-02 21:29:23 +02:00
return nil
}
// readMetaData gets the metadata if it hasn't already been fetched
//
// it also sets the info
2019-09-04 21:00:37 +02:00
func ( o * Object ) readMetaData ( ctx context . Context ) ( err error ) {
2017-10-02 21:29:23 +02:00
if o . hasMetaData {
return nil
}
2019-09-04 21:00:37 +02:00
info , err := o . fs . readMetaDataForPath ( ctx , o . remote , defaultDepth )
2017-10-02 21:29:23 +02:00
if err != nil {
return err
}
return o . setMetaData ( info )
}
// ModTime returns the modification time of the object
//
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
2019-06-17 10:34:30 +02:00
func ( o * Object ) ModTime ( ctx context . Context ) time . Time {
2019-09-04 21:00:37 +02:00
err := o . readMetaData ( ctx )
2017-10-02 21:29:23 +02:00
if err != nil {
fs . Logf ( o , "Failed to read metadata: %v" , err )
return time . Now ( )
}
return o . modTime
}
2023-04-28 18:38:49 +02:00
// Set modified time using propset
//
// <d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns"><d:response><d:href>/ocm/remote.php/webdav/office/wir.jpg</d:href><d:propstat><d:prop><d:lastmodified/></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>
var owncloudPropset = ` < ? xml version = "1.0" encoding = "utf-8" ? >
< D : propertyupdate xmlns : D = "DAV:" >
< D : set >
< D : prop >
< lastmodified xmlns = "DAV:" > % d < / lastmodified >
< / D : prop >
< / D : set >
< / D : propertyupdate >
`
2017-10-02 21:29:23 +02:00
// SetModTime sets the modification time of the local fs object
2019-06-17 10:34:30 +02:00
func ( o * Object ) SetModTime ( ctx context . Context , modTime time . Time ) error {
2023-04-28 18:38:49 +02:00
if o . fs . propsetMtime {
opts := rest . Opts {
Method : "PROPPATCH" ,
Path : o . filePath ( ) ,
NoRedirect : true ,
Body : strings . NewReader ( fmt . Sprintf ( owncloudPropset , modTime . Unix ( ) ) ) ,
}
var result api . Multistatus
var resp * http . Response
var err error
err = o . fs . pacer . Call ( func ( ) ( bool , error ) {
resp , err = o . fs . srv . CallXML ( ctx , & opts , nil , & result )
return o . fs . shouldRetry ( ctx , resp , err )
} )
if err != nil {
if apiErr , ok := err . ( * api . Error ) ; ok {
// does not exist
if apiErr . StatusCode == http . StatusNotFound {
return fs . ErrorObjectNotFound
}
}
return fmt . Errorf ( "couldn't set modified time: %w" , err )
}
// FIXME check if response is valid
if len ( result . Responses ) == 1 && result . Responses [ 0 ] . Props . StatusOK ( ) {
// update cached modtime
o . modTime = modTime
return nil
}
// fallback
return fs . ErrorCantSetModTime
}
2017-10-02 21:29:23 +02:00
return fs . ErrorCantSetModTime
}
// Storable returns a boolean showing whether this object storable
func ( o * Object ) Storable ( ) bool {
return true
}
// Open an object for read
2019-06-17 10:34:30 +02:00
func ( o * Object ) Open ( ctx context . Context , options ... fs . OpenOption ) ( in io . ReadCloser , err error ) {
2017-10-02 21:29:23 +02:00
var resp * http . Response
2020-12-28 12:31:19 +01:00
fs . FixRangeOption ( options , o . size )
2017-10-02 21:29:23 +02:00
opts := rest . Opts {
Method : "GET" ,
Path : o . filePath ( ) ,
Options : options ,
2020-12-26 19:25:19 +01:00
ExtraHeaders : map [ string ] string {
"Depth" : "0" ,
} ,
2017-10-02 21:29:23 +02:00
}
err = o . fs . pacer . Call ( func ( ) ( bool , error ) {
2019-09-04 21:00:37 +02:00
resp , err = o . fs . srv . Call ( ctx , & opts )
2021-03-11 15:44:01 +01:00
return o . fs . shouldRetry ( ctx , resp , err )
2017-10-02 21:29:23 +02:00
} )
if err != nil {
return nil , err
}
return resp . Body , err
}
// Update the object with the contents of the io.Reader, modTime and size
//
2022-08-05 17:35:41 +02:00
// If existing is set then it updates the object rather than creating a new one.
2017-10-02 21:29:23 +02:00
//
// The new object may have been created if an error is returned
2019-06-17 10:34:30 +02:00
func ( o * Object ) Update ( ctx context . Context , in io . Reader , src fs . ObjectInfo , options ... fs . OpenOption ) ( err error ) {
err = o . fs . mkParentDir ( ctx , o . filePath ( ) )
2017-10-02 21:29:23 +02:00
if err != nil {
2021-11-04 11:12:57 +01:00
return fmt . Errorf ( "Update mkParentDir failed: %w" , err )
2017-10-02 21:29:23 +02:00
}
2022-05-20 11:06:55 +02:00
if o . shouldUseChunkedUpload ( src ) {
fs . Debugf ( src , "Update will use the chunked upload strategy" )
err = o . updateChunked ( ctx , in , src , options ... )
if err != nil {
return err
}
} else {
fs . Debugf ( src , "Update will use the normal upload strategy (no chunks)" )
contentType := fs . MimeType ( ctx , src )
filePath := o . filePath ( )
extraHeaders := o . extraHeaders ( ctx , src )
// TODO: define getBody() to enable low-level HTTP/2 retries
err = o . updateSimple ( ctx , in , nil , filePath , src . Size ( ) , contentType , extraHeaders , o . fs . endpointURL , options ... )
if err != nil {
return err
}
2017-10-02 21:29:23 +02:00
}
2022-05-20 11:06:55 +02:00
// read metadata from remote
o . hasMetaData = false
return o . readMetaData ( ctx )
}
func ( o * Object ) extraHeaders ( ctx context . Context , src fs . ObjectInfo ) map [ string ] string {
extraHeaders := map [ string ] string { }
2023-03-13 05:45:54 +01:00
if o . fs . useOCMtime || o . fs . hasOCMD5 || o . fs . hasOCSHA1 {
2019-01-27 14:33:21 +01:00
if o . fs . useOCMtime {
2022-05-20 11:06:55 +02:00
extraHeaders [ "X-OC-Mtime" ] = fmt . Sprintf ( "%d" , src . ModTime ( ctx ) . Unix ( ) )
2019-01-27 14:33:21 +01:00
}
2019-11-24 14:08:06 +01:00
// Set one upload checksum
// Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5
// Nextcloud stores the checksum you supply (SHA1 or MD5) but only stores one
2023-03-13 05:45:54 +01:00
if o . fs . hasOCSHA1 {
2019-06-17 10:34:30 +02:00
if sha1 , _ := src . Hash ( ctx , hash . SHA1 ) ; sha1 != "" {
2022-05-20 11:06:55 +02:00
extraHeaders [ "OC-Checksum" ] = "SHA1:" + sha1
2019-11-24 14:08:06 +01:00
}
}
2022-05-20 11:06:55 +02:00
if o . fs . hasOCMD5 && extraHeaders [ "OC-Checksum" ] == "" {
2019-11-24 14:08:06 +01:00
if md5 , _ := src . Hash ( ctx , hash . MD5 ) ; md5 != "" {
2022-05-20 11:06:55 +02:00
extraHeaders [ "OC-Checksum" ] = "MD5:" + md5
2019-01-27 14:33:21 +01:00
}
2017-10-02 21:29:23 +02:00
}
}
2022-05-20 11:06:55 +02:00
return extraHeaders
}
// Standard update in one request (no chunks)
func ( o * Object ) updateSimple ( ctx context . Context , body io . Reader , getBody func ( ) ( io . ReadCloser , error ) , filePath string , size int64 , contentType string , extraHeaders map [ string ] string , rootURL string , options ... fs . OpenOption ) ( err error ) {
var resp * http . Response
if extraHeaders == nil {
extraHeaders = map [ string ] string { }
}
opts := rest . Opts {
Method : "PUT" ,
Path : filePath ,
GetBody : getBody ,
Body : body ,
NoResponse : true ,
ContentLength : & size , // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
ContentType : contentType ,
Options : options ,
ExtraHeaders : extraHeaders ,
RootURL : rootURL ,
}
2017-10-02 21:29:23 +02:00
err = o . fs . pacer . CallNoRetry ( func ( ) ( bool , error ) {
2019-09-04 21:00:37 +02:00
resp , err = o . fs . srv . Call ( ctx , & opts )
2021-03-11 15:44:01 +01:00
return o . fs . shouldRetry ( ctx , resp , err )
2017-10-02 21:29:23 +02:00
} )
if err != nil {
2018-09-17 09:47:30 +02:00
// Give the WebDAV server a chance to get its internal state in order after the
// error. The error may have been local in which case we closed the connection.
// The server may still be dealing with it for a moment. A sleep isn't ideal but I
// haven't been able to think of a better method to find out if the server has
// finished - ncw
time . Sleep ( 1 * time . Second )
2018-08-16 17:00:30 +02:00
// Remove failed upload
2019-06-17 10:34:30 +02:00
_ = o . Remove ( ctx )
2017-10-02 21:29:23 +02:00
return err
}
2022-05-20 11:06:55 +02:00
return nil
2017-10-02 21:29:23 +02:00
}
// Remove an object
2019-06-17 10:34:30 +02:00
func ( o * Object ) Remove ( ctx context . Context ) error {
2017-10-02 21:29:23 +02:00
opts := rest . Opts {
Method : "DELETE" ,
Path : o . filePath ( ) ,
NoResponse : true ,
}
return o . fs . pacer . Call ( func ( ) ( bool , error ) {
2019-09-04 21:00:37 +02:00
resp , err := o . fs . srv . Call ( ctx , & opts )
2021-03-11 15:44:01 +01:00
return o . fs . shouldRetry ( ctx , resp , err )
2017-10-02 21:29:23 +02:00
} )
}
// Check the interfaces are satisfied
var (
_ fs . Fs = ( * Fs ) ( nil )
_ fs . Purger = ( * Fs ) ( nil )
_ fs . PutStreamer = ( * Fs ) ( nil )
_ fs . Copier = ( * Fs ) ( nil )
_ fs . Mover = ( * Fs ) ( nil )
_ fs . DirMover = ( * Fs ) ( nil )
2019-01-27 17:01:42 +01:00
_ fs . Abouter = ( * Fs ) ( nil )
2017-10-02 21:29:23 +02:00
_ fs . Object = ( * Object ) ( nil )
)