union: fix deadlock when one part of a multi-upload fails

Before this fix, rclone would deadlock when uploading two files at
once, if one errored. This caused the other file to block in the multi
reader and never complete.

This fix drains the input buffer on error which allows the other
upload to complete.

See: https://forum.rclone.org/t/union-with-create-policy-all-copy-stuck-when-first-union-fails/29601
This commit is contained in:
Nick Craig-Wood 2022-03-07 18:21:16 +00:00
parent 6a6d254a9f
commit 5dd51e6149
2 changed files with 10 additions and 0 deletions

View File

@ -4,6 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"sync" "sync"
"time" "time"
@ -84,6 +85,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
err := o.Update(ctx, readers[i], src, options...) err := o.Update(ctx, readers[i], src, options...)
if err != nil { if err != nil {
errs[i] = fmt.Errorf("%s: %w", o.UpstreamFs().Name(), err) errs[i] = fmt.Errorf("%s: %w", o.UpstreamFs().Name(), err)
if len(entries) > 1 {
// Drain the input buffer to allow other uploads to continue
_, _ = io.Copy(ioutil.Discard, readers[i])
}
} }
} else { } else {
errs[i] = fs.ErrorNotAFile errs[i] = fs.ErrorNotAFile

View File

@ -6,6 +6,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"path" "path"
"path/filepath" "path/filepath"
"strings" "strings"
@ -486,6 +487,10 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
} }
if err != nil { if err != nil {
errs[i] = fmt.Errorf("%s: %w", u.Name(), err) errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
if len(upstreams) > 1 {
// Drain the input buffer to allow other uploads to continue
_, _ = io.Copy(ioutil.Discard, readers[i])
}
return return
} }
objs[i] = u.WrapObject(o) objs[i] = u.WrapObject(o)