From 74994a2ec199ced5cc2db2f1628d95113d08fe8f Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 9 Jun 2015 17:29:25 +0100 Subject: [PATCH] Fix uploading big files which was causing timeouts or panics The symtom was one of these two on upload of files only * panic: d.nx != 0 in crypto/md5.(*digest).checkSum * read tcp: i/o timeout It turned out to be a combination of two upstream bugs * https://github.com/mreiferson/go-httpclient/commit/5a2187309eac92103d0329aaae48e06837f260fb * https://groups.google.com/forum/#!topic/golang-dev/0Nl6k5Sj6UU This commit contains a work-around for the second problem, I've fixed the first and had the change accepted upstream. --- fs/accounting.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fs/accounting.go b/fs/accounting.go index 8dc674b2f..5f2dfb8a4 100644 --- a/fs/accounting.go +++ b/fs/accounting.go @@ -187,6 +187,12 @@ func (s *StatsInfo) DoneTransferring(o Object) { // Account limits and accounts for one transfer type Account struct { + // The mutex is to make sure Read() and Close() aren't called + // concurrently. Unfortunately the persistent connection loop + // in http transport calls Read() after Do() returns on + // CancelRequest so this race can happen when it apparently + // shouldn't. + mu sync.Mutex in io.ReadCloser bytes int64 } @@ -200,6 +206,8 @@ func NewAccount(in io.ReadCloser) *Account { // Read bytes from the object - see io.Reader func (file *Account) Read(p []byte) (n int, err error) { + file.mu.Lock() + defer file.mu.Unlock() n, err = file.in.Read(p) file.bytes += int64(n) Stats.Bytes(int64(n)) @@ -215,6 +223,8 @@ func (file *Account) Read(p []byte) (n int, err error) { // Close the object func (file *Account) Close() error { + file.mu.Lock() + defer file.mu.Unlock() // FIXME do something? return file.in.Close() }