storj: implement public link

backend/storj/fs.go

@@ -23,6 +23,7 @@ import (
 	"golang.org/x/text/unicode/norm"
 
 	"storj.io/uplink"
+	"storj.io/uplink/edge"
 )
 
 const (
@@ -161,6 +162,7 @@ var (
 	_ fs.PutStreamer  = &Fs{}
 	_ fs.Mover        = &Fs{}
 	_ fs.Copier       = &Fs{}
+	_ fs.PublicLinker = &Fs{}
 )
 
 // NewFs creates a filesystem backed by Storj.
@@ -545,7 +547,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	defer func() {
 		if err != nil {
 			aerr := upload.Abort()
-			if aerr != nil {
+			if aerr != nil && !errors.Is(aerr, uplink.ErrUploadDone) {
 				fs.Errorf(f, "cp input ./%s %+v: %+v", src.Remote(), options, aerr)
 			}
 		}
@@ -560,6 +562,16 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 	_, err = io.Copy(upload, in)
 	if err != nil {
+		if errors.Is(err, uplink.ErrBucketNotFound) {
+			// Rclone assumes the backend will create the bucket if not existing yet.
+			// Here we create the bucket and return a retry error for rclone to retry the upload.
+			_, err = f.project.EnsureBucket(ctx, bucketName)
+			if err != nil {
+				return nil, err
+			}
+			return nil, fserrors.RetryError(errors.New("bucket was not available, now created, the upload must be retried"))
+		}
+
 		err = fserrors.RetryError(err)
 		fs.Errorf(f, "cp input ./%s %+v: %+v\n", src.Remote(), options, err)
 
@@ -761,3 +773,55 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	// Return the new object
 	return newObjectFromUplink(f, remote, newObject), nil
 }
+
+// PublicLink generates a public link to the remote path (usually readable by anyone)
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+	bucket, key := f.absolute(remote)
+	if len(bucket) == 0 {
+		return "", errors.New("path must be specified")
+	}
+
+	// Rclone requires that a link is only generated if the remote path exists
+	if len(key) == 0 {
+		_, err := f.project.StatBucket(ctx, bucket)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		_, err := f.project.StatObject(ctx, bucket, key)
+		if err != nil {
+			if !errors.Is(err, uplink.ErrObjectNotFound) {
+				return "", err
+			}
+			// No object found, check if there is such a prefix
+			iter := f.project.ListObjects(ctx, bucket, &uplink.ListObjectsOptions{Prefix: key + "/"})
+			if iter.Err() != nil {
+				return "", iter.Err()
+			}
+			if !iter.Next() {
+				return "", err
+			}
+		}
+	}
+
+	sharedPrefix := uplink.SharePrefix{Bucket: bucket, Prefix: key}
+
+	permission := uplink.ReadOnlyPermission()
+	if expire.IsSet() {
+		permission.NotAfter = time.Now().Add(time.Duration(expire))
+	}
+
+	sharedAccess, err := f.access.Share(permission, sharedPrefix)
+	if err != nil {
+		return "", fmt.Errorf("sharing access to object failed: %w", err)
+	}
+
+	creds, err := (&edge.Config{
+		AuthServiceAddress: "auth.storjshare.io:7777",
+	}).RegisterAccess(ctx, sharedAccess, &edge.RegisterAccessOptions{Public: true})
+	if err != nil {
+		return "", fmt.Errorf("creating public link failed: %w", err)
+	}
+
+	return edge.JoinShareURL("https://link.storjshare.io", creds.AccessKeyID, bucket, key, nil)
+}

docs/content/overview.md

@@ -504,7 +504,7 @@ upon backend-specific capabilities.
 | Sia                          | No    | No   | No   | No      | No      | No    | Yes          | No           | No    | Yes      |
 | SMB                          | No    | No   | Yes  | Yes     | No      | No    | Yes          | No           | No    | Yes      |
 | SugarSync                    | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes          | Yes          | No    | Yes      |
-| Storj                        | Yes † | Yes  | Yes  | No      | No      | Yes   | Yes          | No           | No    | No       |
+| Storj                        | Yes † | Yes  | Yes  | No      | No      | Yes   | Yes          | Yes          | No    | No       |
 | Uptobox                      | No    | Yes  | Yes  | Yes     | No      | No    | No           | No           | No    | No       |
 | WebDAV                       | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes ‡        | No           | Yes   | Yes      |
 | Yandex Disk                  | Yes   | Yes  | Yes  | Yes     | Yes     | No    | Yes          | Yes          | Yes   | Yes      |

fstest/fstests/fstests.go

@@ -1734,8 +1734,10 @@ func Run(t *testing.T, opt *Opt) {
 					// ensure sub remote isn't empty
 					buf := bytes.NewBufferString("somecontent")
 					obji := object.NewStaticObjectInfo("somefile", time.Now(), int64(buf.Len()), true, nil, nil)
-					_, err = subRemote.Put(ctx, buf, obji)
+					retry(t, "Put", func() error {
-					require.NoError(t, err)
+						_, err := subRemote.Put(ctx, buf, obji)
+						return err
+					})
 
 					link4, err := wrapPublicLinkFunc(subRemote.Features().PublicLink)(ctx, "", expiry, false)
 					require.NoError(t, err, "Sharing root in a sub-remote should work")