ytmusic: initialize and develop oauth process

backend/all/all.go

@@ -58,5 +58,6 @@ import (
 	_ "github.com/rclone/rclone/backend/uptobox"
 	_ "github.com/rclone/rclone/backend/webdav"
 	_ "github.com/rclone/rclone/backend/yandex"
+	_ "github.com/rclone/rclone/backend/youtubemusic"
 	_ "github.com/rclone/rclone/backend/zoho"
 )

backend/youtubemusic/api/types.go

@@ -0,0 +1,37 @@
+// Package api provides types used by the YouTube Music API.
+package api
+
+// OAuthTVAndLimitedDeviceRequest represents the JSON API object that's sent to the oauth API endpoint.
+type OAuthTVAndLimitedDeviceRequest struct {
+	Scope    string `json:"scope"`
+	ClientID string `json:"client_id"`
+}
+
+// OAuthTVAndLimitedDeviceResponse represents the JSON API object that's received from the oauth API endpoint.
+type OAuthTVAndLimitedDeviceResponse struct {
+	DeviceCode      string `json:"device_code"`
+	UserCode        string `json:"user_code"`
+	ExpiresIn       int    `json:"expires_in"`
+	Interval        int    `json:"interval"`
+	VerificationURL string `json:"verification_url"`
+	Error           string `json:"error"`
+}
+
+// OAuthTokenTVAndLimitedDeviceRequest represents the JSON API object that's sent to the oauth token API endpoint.
+type OAuthTokenTVAndLimitedDeviceRequest struct {
+	ClientID     string `json:"client_id"`
+	ClientSecret string `json:"client_secret"`
+	GrantType    string `json:"grant_type"`
+	Code         string `json:"code"`
+}
+
+// OAuthTokenTVAndLimitedDeviceResponse represents the JSON API object that's received from the oauth token API endpoint.
+type OAuthTokenTVAndLimitedDeviceResponse struct {
+	Scope            string `json:"scope"`
+	TokenType        string `json:"token_type"`
+	AccessToken      string `json:"access_token"`
+	RefreshToken     string `json:"refresh_token"`
+	ExpiresIn        int    `json:"expires_in"`
+	Error            string `json:"error"`
+	ErrorDescription string `json:"error_description"`
+}

backend/youtubemusic/fs.go

@@ -0,0 +1,121 @@
+package youtubemusic
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/oauthutil"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+type Fs struct {
+	name      string                 // name of this remote
+	root      string                 // the path we are working on if any
+	opt       Options                // parsed options
+	features  *fs.Features           // optional features
+	unAuth    *rest.Client           // unauthenticated http client
+	srv       *rest.Client           // the connection to the server
+	ts        *oauthutil.TokenSource // token source for oauth2
+	pacer     *fs.Pacer              // To pace the API calls
+	startTime time.Time              // time Fs was started - used for datestamps
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	return fmt.Sprintf("YouTube Music path '%q'", f.root)
+}
+
+// Precision returns the precision
+func (f *Fs) Precision() time.Duration {
+	return fs.ModTimeNotSupported
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.None)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	// TODO:
+	return nil, nil
+}
+
+// NewObject finds the Object at remote.  If it can't be found
+// it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	// TODO:
+	return nil, nil
+}
+
+// Put the object into the bucket
+//
+// Copy the reader in to the new object which is returned.
+//
+// The new object may have been created if an error is returned
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	// TODO:
+	return nil, nil
+}
+
+// Mkdir creates the album if it doesn't exist
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	// TODO:
+	return nil
+}
+
+// Rmdir deletes the bucket if the fs is at the root
+//
+// Returns an error if it isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	// TODO:
+	return nil
+}
+
+// NewFs constructs an Fs from the path.
+//
+// The returned Fs implements fs.Fs interface.
+func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, error) {
+	// Parse config into Options struct
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	fsys := &Fs{}
+
+	return fsys, nil
+}
+
+// Check the interfaces are satisfied.
+var _ fs.Fs = &Fs{}

backend/youtubemusic/object.go

@@ -0,0 +1,142 @@
+package youtubemusic
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/log"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+// Object describes a youtubemusic object
+type Object struct {
+	fs       *Fs       // what this object is part of
+	remote   string    // The remote path
+	url      string    // download path
+	id       string    // ID of this object
+	bytes    int64     // Bytes in the object
+	modTime  time.Time // Modified time of the object
+	mimeType string
+}
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	defer log.Trace(o, "")("")
+	err := o.readMetaData(ctx)
+	if err != nil {
+		fs.Debugf(o, "ModTime: Failed to read metadata: %v", err)
+		return time.Now()
+	}
+	return o.modTime
+}
+
+// Size returns the size of an object in bytes
+func (o *Object) Size() int64 {
+	// TODO:
+	return 0
+}
+
+// Hash returns the Md5sum of an object returning a lowercase hex string
+func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
+	return "", hash.ErrUnsupported
+
+}
+
+// Storable returns a boolean as to whether this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
+	return fs.ErrorCantSetModTime
+}
+
+// Open an object for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	defer log.Trace(o, "")("")
+	err = o.readMetaData(ctx)
+	if err != nil {
+		fs.Debugf(o, "Open: Failed to read metadata: %v", err)
+		return nil, err
+	}
+	var resp *http.Response
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: o.downloadURL(),
+		Options: options,
+	}
+	err = o.fs.pacer.Call(func() (bool, error) {
+		resp, err = o.fs.srv.Call(ctx, &opts)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return resp.Body, err
+}
+
+// Update the object with the contents of the io.Reader, modTime and size
+//
+// The new object may have been created if an error is returned
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	// TODO:
+	return nil
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+	// TODO:
+	return nil
+}
+
+// readMetaData gets the metadata if it hasn't already been fetched
+//
+// it also sets the info
+func (o *Object) readMetaData(ctx context.Context) (err error) {
+	// TODO:
+	return nil
+}
+
+// setMetaData sets the fs data from a storage.Object
+func (o *Object) setMetaData() {
+	// TODO:
+}
+
+// downloadURL returns the URL for a full bytes download for the object
+func (o *Object) downloadURL() (url string) {
+	// TODO:
+	// url := o.url + "=d"
+	// if strings.HasPrefix(o.mimeType, "video/") {
+	// 	url += "v"
+	// }
+	return url
+}
+
+// Check the interfaces are satisfied
+var _ fs.Object = &Object{}

backend/youtubemusic/youtubemusic.go

@@ -0,0 +1,248 @@
+// Package youtubemusic provides the youtubemusic backend.
+package youtubemusic
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/rclone/rclone/backend/youtubemusic/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/lib/oauthutil"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/skratchdot/open-golang/open"
+	"golang.org/x/oauth2"
+)
+
+const (
+	// The app for these API keys should be for TV and Limited-Input Device.
+	rcloneClientID              = "861556708454-d6dlm3lh05idd8npek18k6be8ba3oc68.apps.googleusercontent.com" // TODO: update this
+	rcloneEncryptedClientSecret = "aCVz_k_XoJc9gc3XuuDCeq2VzXsQFd4QTKsyF8ir2Bgnj5abr28JQw"                   // TODO: update this
+
+	// These 2 scopes are the only YouTube Data API scopes available for TV and Limited-Input Device.
+	// source: https://developers.google.com/youtube/v3/guides/auth/devices#allowedscopes
+	scopeYoutubeReadWrite = "https://www.googleapis.com/auth/youtube"          // manage your YouTube account
+	scopeYoutubeReadOnly  = "https://www.googleapis.com/auth/youtube.readonly" // view your YouTube account
+
+	ytmusicDomain    = "https://music.youtube.com"
+	ytmusicBaseAPI   = ytmusicDomain + "/youtubei/v1/"
+	ytmusicParams    = "?alt=json"
+	ytmusicParamsKey = "&key=AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30"
+	userAgent        = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"
+
+	oauthCodeURL   = "https://www.youtube.com/o/oauth2/device/code"
+	oauthTokenURL  = "https://oauth2.googleapis.com/token"
+	oauthUserAgent = userAgent + " Cobalt/Version"
+)
+
+var (
+	oauthScope string
+	deviceCode string
+)
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "youtube music",
+		Prefix:      "ytmusic",
+		Description: "YouTube Music",
+		NewFs:       NewFs,
+		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
+			// Parse config into Options struct
+			opt := new(Options)
+			err := configstruct.Set(m, opt)
+			if err != nil {
+				return nil, fmt.Errorf("couldn't parse config into struct: %w", err)
+			}
+
+			switch config.State {
+			case "":
+				// Fill in the scopes
+				if opt.ReadOnly {
+					oauthScope = scopeYoutubeReadOnly
+				} else {
+					oauthScope = scopeYoutubeReadWrite
+				}
+
+				// Update client_id and client_secret if they are empty
+				if val, _ := m.Get("client_id"); val == "" {
+					m.Set("client_id", rcloneClientID)
+				}
+				if val, _ := m.Get("client_secret"); val == "" {
+					m.Set("client_secret", obscure.MustReveal(rcloneEncryptedClientSecret))
+				}
+
+				// Post the response from oauthCodeURL
+				clientID, _ := m.Get("client_id")
+				oauthTVAndLimitedDevice, err := postOAuthTVAndLimitedDevice(ctx, api.OAuthTVAndLimitedDeviceRequest{
+					Scope:    oauthScope,
+					ClientID: clientID,
+				})
+				if err != nil {
+					return nil, fmt.Errorf("failed to postOAuthCodeURL: %w", err)
+				}
+				deviceCode = oauthTVAndLimitedDevice.DeviceCode
+
+				// Open the verification URL in the browser
+				url := fmt.Sprintf("%s?user_code=%s", oauthTVAndLimitedDevice.VerificationURL, oauthTVAndLimitedDevice.UserCode)
+				open.Start(url)
+
+				return fs.ConfigConfirm("config_auth_do", true, "config_init", fmt.Sprintf("Go to %s, finish the login flow and press Enter when done, Ctrl-C to abort", url))
+			case "config_auth_do": // Continue the authentication process
+				// Post the response from oauthTokenURL
+				clientID, _ := m.Get("client_id")
+				clientSecret, _ := m.Get("client_secret")
+				oauthTokenTVAndLimitedDevice, err := postOAuthTokenTVAndLimitedDevice(ctx, api.OAuthTokenTVAndLimitedDeviceRequest{
+					ClientID:     clientID,
+					ClientSecret: clientSecret,
+					GrantType:    "http://oauth.net/grant_type/device/1.0",
+					Code:         deviceCode,
+				})
+				if err != nil {
+					return nil, fmt.Errorf("failed to postOAuthTokenURL: %w", err)
+				}
+
+				// Save the token to the config file
+				err = oauthutil.PutToken(name, m, &oauth2.Token{
+					AccessToken:  oauthTokenTVAndLimitedDevice.AccessToken,
+					RefreshToken: oauthTokenTVAndLimitedDevice.RefreshToken,
+					TokenType:    oauthTokenTVAndLimitedDevice.TokenType,
+					Expiry:       time.Now().Add(time.Duration(oauthTokenTVAndLimitedDevice.ExpiresIn) * time.Second),
+				}, true)
+				if err != nil {
+					return nil, fmt.Errorf("error while saving token: %w", err)
+				}
+
+				return nil, nil
+			}
+			return nil, fmt.Errorf("unknown state %q", config.State)
+		},
+		Options: append(oauthutil.SharedOptions, []fs.Option{{
+			Name:    "read_only",
+			Default: false,
+			Help: `Set to make the YouTube Music backend read only.
+
+If you choose read only then rclone will only request read only access
+to your music, otherwise rclone will request full access.`,
+		}, {
+			Name: "playlist-id",
+			Help: "ID of the playlist to sync. Can be found in the playlist's URL.",
+		}}...),
+	})
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+	ReadOnly bool `config:"read_only"`
+
+	// ReadSize        bool                 `config:"read_size"`
+	// StartYear       int                  `config:"start_year"`
+	// IncludeArchived bool                 `config:"include_archived"`
+	// Enc             encoder.MultiEncoder `config:"encoding"`
+	// BatchMode       string               `config:"batch_mode"`
+	// BatchSize       int                  `config:"batch_size"`
+	// BatchTimeout    fs.Duration          `config:"batch_timeout"`
+}
+
+// ------------------------------------------------------------
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+	429, // Too Many Requests.
+	500, // Internal Server Error
+	502, // Bad Gateway
+	503, // Service Unavailable
+	504, // Gateway Timeout
+	509, // Bandwidth Limit Exceeded
+}
+
+// shouldRetry returns a boolean as to whether this resp and err
+// deserve to be retried.  It returns the err as a convenience
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+}
+
+// postOAuthTVAndLimitedDevice sends a POST request to the oauthCodeURL and returns the response.
+func postOAuthTVAndLimitedDevice(ctx context.Context, reqData api.OAuthTVAndLimitedDeviceRequest) (resp api.OAuthTVAndLimitedDeviceResponse, err error) {
+	// Create a new HTTP client with the provided context.
+	httpClient := fshttp.NewClient(ctx)
+
+	// Marshal the request data into JSON.
+	reqStr, err := json.Marshal(reqData)
+	if err != nil {
+		return resp, fmt.Errorf("failed to marshal reqData: %w", err)
+	}
+
+	// Send a POST request to the oauthCodeURL with the request data.
+	respHTTP, err := httpClient.Post(oauthCodeURL, "application/json", bytes.NewBuffer(reqStr))
+	if err != nil {
+		return resp, fmt.Errorf("failed to get oauth code: %w", err)
+	}
+	defer fs.CheckClose(respHTTP.Body, &err)
+
+	// Read the response body.
+	respBody, err := rest.ReadBody(respHTTP)
+	if err != nil {
+		return resp, fmt.Errorf("failed to read respBody: %w", err)
+	}
+
+	// Unmarshal the response body into a struct.
+	response := new(api.OAuthTVAndLimitedDeviceResponse)
+	err = json.Unmarshal(respBody, &response)
+	if err != nil {
+		return resp, fmt.Errorf("failed to parse respBody: %w", err)
+	}
+	if response.Error != "" {
+		return resp, fmt.Errorf("failed to get oauth code: %s", respBody)
+	}
+
+	return *response, nil
+}
+
+// postOAuthTokenTVAndLimitedDevice sends a POST request to the oauthTokenURL and returns the response.
+func postOAuthTokenTVAndLimitedDevice(ctx context.Context, reqData api.OAuthTokenTVAndLimitedDeviceRequest) (resp api.OAuthTokenTVAndLimitedDeviceResponse, err error) {
+	// Create a new HTTP client with the provided context.
+	httpClient := fshttp.NewClient(ctx)
+
+	// Marshal the request data into JSON.
+	reqStr, err := json.Marshal(reqData)
+	if err != nil {
+		return resp, fmt.Errorf("failed to marshal reqData: %w", err)
+	}
+
+	// Send a POST request to the oauthTokenURL with the request data.
+	respHTTP, err := httpClient.Post(oauthTokenURL, "application/json", bytes.NewBuffer(reqStr))
+	if err != nil {
+		return resp, fmt.Errorf("failed to get oauth token: %w", err)
+	}
+	defer fs.CheckClose(respHTTP.Body, &err)
+
+	// Read the response body.
+	respBody, err := rest.ReadBody(respHTTP)
+	if err != nil {
+		return resp, fmt.Errorf("failed to read respBody: %w", err)
+	}
+
+	// Unmarshal the response body into a struct.
+	response := new(api.OAuthTokenTVAndLimitedDeviceResponse)
+	err = json.Unmarshal(respBody, &response)
+	if err != nil {
+		return resp, fmt.Errorf("failed to parse respBody: %w", err)
+	}
+	if response.Error != "" {
+		return resp, fmt.Errorf("failed to get oauth token: %s", respBody)
+	}
+
+	return *response, nil
+}