http: add --http-no-head to stop rclone doing HEAD in listings #3523

backend/http/http.go

@@ -78,6 +78,26 @@ Note that this may cause rclone to confuse genuine HTML files with
 directories.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "no_head",
+			Help: `Don't use HEAD requests to find file sizes in dir listing
+
+If your site is being very slow to load then you can try this option.
+Normally rclone does a HEAD request for each potential file in a
+directory listing to:
+
+- find its size
+- check it really exists
+- check to see if it is a directory
+
+If you set this option, rclone will not do the HEAD request.  This will mean
+
+- directory listings are much quicker
+- rclone won't have the times or sizes of any files
+- some files that don't exist may be in the listing
+`,
+			Default:  false,
+			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -87,6 +107,7 @@ directories.`,
 type Options struct {
 	Endpoint string          `config:"url"`
 	NoSlash  bool            `config:"no_slash"`
+	NoHead   bool            `config:"no_head"`
 	Headers  fs.CommaSepList `config:"headers"`
 }
 
@@ -516,6 +537,12 @@ func (o *Object) url() string {
 
 // stat updates the info field in the Object
 func (o *Object) stat(ctx context.Context) error {
+	if o.fs.opt.NoHead {
+		o.size = -1
+		o.modTime = timeUnset
+		o.contentType = fs.MimeType(ctx, o)
+		return nil
+	}
 	url := o.url()
 	req, err := http.NewRequest("HEAD", url, nil)
 	if err != nil {