extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-22 00:13:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

ulozto: revert the temporary file size limitations

Browse Source
This commit is contained in:
IoT Maestro 2024-03-28 08:09:36 +00:00 committed by Nick Craig-Wood
parent 748c43d525
commit c9ce384ec7
2 changed files with 1 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/ulozto/ulozto.go
View File

@ -38,8 +38,6 @@ const (
maxSleep = 2 * time.Second
decayConstant = 2 // bigger for slower decay, exponential
rootURL = "https://apis.uloz.to"
// TODO temporary limitation, remove with chunked upload impl
maxFileSizeBytes = 2500 * 1024 * 1024
)
// Options defines the configuration for this backend
@ -194,9 +192,7 @@ func errorHandler(resp *http.Response) error {
// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
429, // Too Many Requests.
// TODO: random 500s should be retried but the error code corresponds to a known issue with uploading large files,
// leading to numerous (slow & resource consuming) retries. Don't retry them until the root cause is addressed.
// 500, // Internal Server Error
500, // Internal Server Error
502, // Bad Gateway
503, // Service Unavailable
504, // Gateway Timeout
@ -447,10 +443,6 @@ func (f *Fs) uploadUnchecked(ctx context.Context, name, parentSlug string, info
// Put implements the mandatory method fs.Fs.Put.
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
// TODO: workaround for uloz.to's bug. Remove when chunked upload support is implemented.
if src.Size() > maxFileSizeBytes {
return nil, errors.New("file size over the supported max threshold")
}
existingObj, err := f.NewObject(ctx, src.Remote())
switch {

31
backend/ulozto/ulozto_test.go
View File

@ -1,7 +1,6 @@
package ulozto
import (
"bytes"
"context"
"errors"
"testing"
@ -10,7 +9,6 @@ import (
"github.com/rclone/rclone/backend/ulozto/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/require"
@ -86,32 +84,3 @@ func TestListWithoutMetadata(t *testing.T) {
// Tear down
require.NoError(t, operations.Purge(ctx, f, ""))
}
// TestUploadLargeFile verifies that files over the supported threshold are not uploaded.
func TestUploadLargeFile(t *testing.T) {
const (
remoteName = "TestUlozto:"
payload = "foobar"
filesize = maxFileSizeBytes + 1
)
ctx := context.Background()
fstest.Initialise()
subRemoteName, subRemoteLeaf, err := fstest.RandomRemoteName(remoteName)
require.NoError(t, err)
f, err := fs.NewFs(ctx, subRemoteName)
if errors.Is(err, fs.ErrorNotFoundInConfigFile) {
t.Logf("Didn't find %q in config file - skipping tests", remoteName)
return
}
require.NoError(t, err)
file := fstest.Item{ModTime: time.UnixMilli(123456789), Path: subRemoteLeaf, Size: int64(filesize)}
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
// The payload buffer is just a placeholder which shouldn't be used
_, err = f.Put(ctx, bytes.NewBufferString(payload), obji)
require.Error(t, err, "File size over the supported max threshold.")
// Verify the remote stayed intact
fstest.CheckListing(t, f, []fstest.Item{})
}