// Package oauthutil provides OAuth utilities. package oauthutil import ( "context" "encoding/json" "errors" "fmt" "html/template" "net" "net/http" "net/url" "strings" "sync" "time" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/lib/random" "github.com/skratchdot/open-golang/open" "golang.org/x/oauth2" ) const ( // TitleBarRedirectURL is the OAuth2 redirect URL to use when the authorization // code should be returned in the title bar of the browser, with the page text // prompting the user to copy the code and paste it in the application. TitleBarRedirectURL = "urn:ietf:wg:oauth:2.0:oob" // bindPort is the port that we bind the local webserver to bindPort = "53682" // bindAddress is binding for local webserver when active bindAddress = "127.0.0.1:" + bindPort // RedirectURL is redirect to local webserver when active RedirectURL = "http://" + bindAddress + "/" // RedirectPublicURL is redirect to local webserver when active with public name RedirectPublicURL = "http://localhost.rclone.org:" + bindPort + "/" // RedirectLocalhostURL is redirect to local webserver when active with localhost RedirectLocalhostURL = "http://localhost:" + bindPort + "/" // RedirectPublicSecureURL is a public https URL which // redirects to the local webserver RedirectPublicSecureURL = "https://oauth.rclone.org/" // AuthResponseTemplate is a template to handle the redirect URL for oauth requests AuthResponseTemplate = `
{{ if eq .OK false }} Error: {{ .Name }}` ) // SharedOptions are shared between backends the utilize an OAuth flow var SharedOptions = []fs.Option{{ Name: config.ConfigClientID, Help: "OAuth Client Id.\n\nLeave blank normally.", }, { Name: config.ConfigClientSecret, Help: "OAuth Client Secret.\n\nLeave blank normally.", }, { Name: config.ConfigToken, Help: "OAuth Access Token as a JSON blob.", Advanced: true, }, { Name: config.ConfigAuthURL, Help: "Auth server URL.\n\nLeave blank to use the provider defaults.", Advanced: true, }, { Name: config.ConfigTokenURL, Help: "Token server url.\n\nLeave blank to use the provider defaults.", Advanced: true, }} // oldToken contains an end-user's tokens. // This is the data you must store to persist authentication. // // From the original code.google.com/p/goauth2/oauth package - used // for backwards compatibility in the rclone config file type oldToken struct { AccessToken string RefreshToken string Expiry time.Time } // GetToken returns the token saved in the config file under // section name. func GetToken(name string, m configmap.Mapper) (*oauth2.Token, error) { tokenString, ok := m.Get(config.ConfigToken) if !ok || tokenString == "" { return nil, fmt.Errorf("empty token found - please run \"rclone config reconnect %s:\"", name) } token := new(oauth2.Token) err := json.Unmarshal([]byte(tokenString), token) if err != nil { return nil, err } // if has data then return it if token.AccessToken != "" { return token, nil } // otherwise try parsing as oldToken oldtoken := new(oldToken) err = json.Unmarshal([]byte(tokenString), oldtoken) if err != nil { return nil, err } // Fill in result into new token token.AccessToken = oldtoken.AccessToken token.RefreshToken = oldtoken.RefreshToken token.Expiry = oldtoken.Expiry // Save new format in config file err = PutToken(name, m, token, false) if err != nil { return nil, err } return token, nil } // PutToken stores the token in the config file // // This saves the config file if it changes func PutToken(name string, m configmap.Mapper, token *oauth2.Token, newSection bool) error { tokenBytes, err := json.Marshal(token) if err != nil { return err } tokenString := string(tokenBytes) old, ok := m.Get(config.ConfigToken) if !ok || tokenString != old { m.Set(config.ConfigToken, tokenString) fs.Debugf(name, "Saved new token in config file") } return nil } // TokenSource stores updated tokens in the config file type TokenSource struct { mu sync.Mutex name string m configmap.Mapper tokenSource oauth2.TokenSource token *oauth2.Token config *oauth2.Config ctx context.Context expiryTimer *time.Timer // signals whenever the token expires } // If token has expired then first try re-reading it (and its refresh token) // from the config file in case a concurrently running rclone has updated them // already. // Returns whether either of the two tokens has been reread. func (ts *TokenSource) reReadToken() (changed bool) { tokenString, found := ts.m.Get(config.ConfigToken) if !found || tokenString == "" { fs.Debugf(ts.name, "Failed to read token out of config file") return false } newToken := new(oauth2.Token) err := json.Unmarshal([]byte(tokenString), newToken) if err != nil { fs.Debugf(ts.name, "Failed to parse token out of config file: %v", err) return false } if !newToken.Valid() { fs.Debugf(ts.name, "Loaded invalid token from config file - ignoring") } else { fs.Debugf(ts.name, "Loaded fresh token from config file") changed = true } if newToken.RefreshToken != "" && newToken.RefreshToken != ts.token.RefreshToken { fs.Debugf(ts.name, "Loaded new refresh token from config file") changed = true } if changed { ts.token = newToken ts.tokenSource = nil // invalidate since we changed the token } return changed } // Token returns a token or an error. // Token must be safe for concurrent use by multiple goroutines. // The returned Token must not be modified. // // This saves the token in the config file if it has changed func (ts *TokenSource) Token() (*oauth2.Token, error) { ts.mu.Lock() defer ts.mu.Unlock() var ( token *oauth2.Token err error changed = false ) const maxTries = 5 // Try getting the token a few times for i := 1; i <= maxTries; i++ { // Try reading the token from the config file in case it has // been updated by a concurrent rclone process if !ts.token.Valid() { if ts.reReadToken() { changed = true } else if ts.token.RefreshToken == "" { return nil, fserrors.FatalError( fmt.Errorf("token expired and there's no refresh token - manually refresh with \"rclone config reconnect %s:\"", ts.name), ) } } // Make a new token source if required if ts.tokenSource == nil { ts.tokenSource = ts.config.TokenSource(ts.ctx, ts.token) } token, err = ts.tokenSource.Token() if err == nil { break } fs.Debugf(ts.name, "Token refresh failed try %d/%d: %v", i, maxTries, err) time.Sleep(1 * time.Second) } if err != nil { return nil, fmt.Errorf("couldn't fetch token - maybe it has expired? - refresh with \"rclone config reconnect %s:\": %w", ts.name, err) } changed = changed || (*token != *ts.token) ts.token = token if changed { // Bump on the expiry timer if it is set if ts.expiryTimer != nil { ts.expiryTimer.Reset(ts.timeToExpiry()) } err = PutToken(ts.name, ts.m, token, false) if err != nil { return nil, fmt.Errorf("couldn't store token: %w", err) } } return token, nil } // Invalidate invalidates the token func (ts *TokenSource) Invalidate() { ts.mu.Lock() ts.token.AccessToken = "" ts.mu.Unlock() } // Expire marks the token as expired // // This also marks the token in the config file as expired, if it is the same one func (ts *TokenSource) Expire() error { ts.mu.Lock() defer ts.mu.Unlock() ts.token.Expiry = time.Now().Add(time.Hour * (-1)) // expire token t, err := GetToken(ts.name, ts.m) if err != nil { return err } if t.AccessToken == ts.token.AccessToken { err = PutToken(ts.name, ts.m, ts.token, false) } return err } // timeToExpiry returns how long until the token expires // // Call with the lock held func (ts *TokenSource) timeToExpiry() time.Duration { t := ts.token if t == nil { return 0 } if t.Expiry.IsZero() { return 3e9 * time.Second // ~95 years } return time.Until(t.Expiry) } // OnExpiry returns a channel which has the time written to it when // the token expires. Note that there is only one channel so if // attaching multiple go routines it will only signal to one of them. func (ts *TokenSource) OnExpiry() <-chan time.Time { ts.mu.Lock() defer ts.mu.Unlock() if ts.expiryTimer == nil { ts.expiryTimer = time.NewTimer(ts.timeToExpiry()) } return ts.expiryTimer.C } // Check interface satisfied var _ oauth2.TokenSource = (*TokenSource)(nil) // Context returns a context with our HTTP Client baked in for oauth2 func Context(ctx context.Context, client *http.Client) context.Context { return context.WithValue(ctx, oauth2.HTTPClient, client) } // overrideCredentials sets the ClientID and ClientSecret from the // config file if they are not blank. // If any value is overridden, true is returned. // the origConfig is copied func overrideCredentials(name string, m configmap.Mapper, origConfig *oauth2.Config) (newConfig *oauth2.Config, changed bool) { newConfig = new(oauth2.Config) *newConfig = *origConfig changed = false ClientID, ok := m.Get(config.ConfigClientID) if ok && ClientID != "" { newConfig.ClientID = ClientID changed = true } ClientSecret, ok := m.Get(config.ConfigClientSecret) if ok && ClientSecret != "" { newConfig.ClientSecret = ClientSecret changed = true } AuthURL, ok := m.Get(config.ConfigAuthURL) if ok && AuthURL != "" { newConfig.Endpoint.AuthURL = AuthURL changed = true } TokenURL, ok := m.Get(config.ConfigTokenURL) if ok && TokenURL != "" { newConfig.Endpoint.TokenURL = TokenURL changed = true } return newConfig, changed } // NewClientWithBaseClient gets a token from the config file and // configures a Client with it. It returns the client and a // TokenSource which Invalidate may need to be called on. It uses the // httpClient passed in as the base client. func NewClientWithBaseClient(ctx context.Context, name string, m configmap.Mapper, config *oauth2.Config, baseClient *http.Client) (*http.Client, *TokenSource, error) { config, _ = overrideCredentials(name, m, config) token, err := GetToken(name, m) if err != nil { return nil, nil, err } // Set our own http client in the context ctx = Context(ctx, baseClient) // Wrap the TokenSource in our TokenSource which saves changed // tokens in the config file ts := &TokenSource{ name: name, m: m, token: token, config: config, ctx: ctx, } return oauth2.NewClient(ctx, ts), ts, nil } // NewClient gets a token from the config file and configures a Client // with it. It returns the client and a TokenSource which Invalidate may need to be called on func NewClient(ctx context.Context, name string, m configmap.Mapper, oauthConfig *oauth2.Config) (*http.Client, *TokenSource, error) { return NewClientWithBaseClient(ctx, name, m, oauthConfig, fshttp.NewClient(ctx)) } // AuthResult is returned from the web server after authorization // success or failure type AuthResult struct { OK bool // Failure or Success? Name string Description string Code string HelpURL string Form url.Values // the complete contents of the form Err error // any underlying error to report } // Error satisfies the error interface so AuthResult can be used as an error func (ar *AuthResult) Error() string { status := "Error" if ar.OK { status = "OK" } return fmt.Sprintf("%s: %s\nCode: %q\nDescription: %s\nHelp: %s", status, ar.Name, ar.Code, ar.Description, ar.HelpURL) } // CheckAuthFn is called when a good Auth has been received type CheckAuthFn func(*oauth2.Config, *AuthResult) error // Options for the oauth config type Options struct { OAuth2Config *oauth2.Config // Basic config for oauth2 NoOffline bool // If set then "access_type=offline" parameter is not passed CheckAuth CheckAuthFn // When the AuthResult is known the checkAuth function is called if set OAuth2Opts []oauth2.AuthCodeOption // extra oauth2 options StateBlankOK bool // If set, state returned as "" is deemed to be OK } // ConfigOut returns a config item suitable for the backend config // // state is the place to return the config to // oAuth is the config to run the oauth with func ConfigOut(state string, oAuth *Options) (*fs.ConfigOut, error) { return &fs.ConfigOut{ State: state, OAuth: oAuth, }, nil } // ConfigOAuth does the oauth config specified in the config block // // This is called with a state which has pushed on it // // state prefixed with "*oauth" // state for oauth to return to // state that returned the OAuth when we wish to recall it // value that returned the OAuth func ConfigOAuth(ctx context.Context, name string, m configmap.Mapper, ri *fs.RegInfo, in fs.ConfigIn) (*fs.ConfigOut, error) { stateParams, state := fs.StatePop(in.State) // Make the next state newState := func(state string) string { return fs.StatePush(stateParams, state) } // Recall the Oauth state again by calling the Config with the same input again getOAuth := func() (opt *Options, err error) { tmpState, _ := fs.StatePop(stateParams) tmpState, State := fs.StatePop(tmpState) _, Result := fs.StatePop(tmpState) out, err := ri.Config(ctx, name, m, fs.ConfigIn{State: State, Result: Result}) if err != nil { return nil, err } if out.OAuth == nil { return nil, errors.New("failed to recall OAuth state") } opt, ok := out.OAuth.(*Options) if !ok { return nil, fmt.Errorf("internal error: oauth failed: wrong type in config: %T", out.OAuth) } if opt.OAuth2Config == nil { return nil, errors.New("internal error: oauth failed: OAuth2Config not set") } return opt, nil } switch state { case "*oauth": // See if already have a token tokenString, ok := m.Get("token") if ok && tokenString != "" { return fs.ConfigConfirm(newState("*oauth-confirm"), true, "config_refresh_token", "Already have a token - refresh?") } return fs.ConfigGoto(newState("*oauth-confirm")) case "*oauth-confirm": if in.Result == "false" { return fs.ConfigGoto(newState("*oauth-done")) } return fs.ConfigConfirm(newState("*oauth-islocal"), true, "config_is_local", "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n") case "*oauth-islocal": if in.Result == "true" { return fs.ConfigGoto(newState("*oauth-do")) } return fs.ConfigGoto(newState("*oauth-remote")) case "*oauth-remote": opt, err := getOAuth() if err != nil { return nil, err } if noWebserverNeeded(opt.OAuth2Config) { authURL, _, err := getAuthURL(name, m, opt.OAuth2Config, opt) if err != nil { return nil, err } return fs.ConfigInput(newState("*oauth-do"), "config_verification_code", fmt.Sprintf("Verification code\n\nGo to this URL, authenticate then paste the code here.\n\n%s\n", authURL)) } var out strings.Builder fmt.Fprintf(&out, `For this to work, you will need rclone available on a machine that has a web browser available. For more help and alternate methods see: https://rclone.org/remote_setup/ Execute the following on the machine with the web browser (same rclone version recommended): `) // Find the overridden options inM := ri.Options.NonDefault(m) delete(inM, fs.ConfigToken) // delete token as we are refreshing it for k, v := range inM { fs.Debugf(nil, "sending %s = %q", k, v) } // Encode them into a string mCopyString, err := inM.Encode() if err != nil { return nil, fmt.Errorf("oauthutil authorize encode: %w", err) } // Write what the user has to do if len(mCopyString) > 0 { fmt.Fprintf(&out, "\trclone authorize %q %q\n", ri.Name, mCopyString) } else { fmt.Fprintf(&out, "\trclone authorize %q\n", ri.Name) } fmt.Fprintln(&out, "\nThen paste the result.") return fs.ConfigInput(newState("*oauth-authorize"), "config_token", out.String()) case "*oauth-authorize": // Read the updates to the config outM := configmap.Simple{} token := oauth2.Token{} code := in.Result newFormat := true err := outM.Decode(code) if err != nil { newFormat = false err = json.Unmarshal([]byte(code), &token) } if err != nil { return fs.ConfigError(newState("*oauth-authorize"), fmt.Sprintf("Couldn't decode response - try again (make sure you are using a matching version of rclone on both sides: %v\n", err)) } // Save the config updates if newFormat { for k, v := range outM { m.Set(k, v) fs.Debugf(nil, "received %s = %q", k, v) } } else { m.Set(fs.ConfigToken, code) } return fs.ConfigGoto(newState("*oauth-done")) case "*oauth-do": code := in.Result opt, err := getOAuth() if err != nil { return nil, err } oauthConfig, changed := overrideCredentials(name, m, opt.OAuth2Config) if changed { fs.Logf(nil, "Make sure your Redirect URL is set to %q in your custom config.\n", oauthConfig.RedirectURL) } if code == "" { oauthConfig = fixRedirect(oauthConfig) code, err = configSetup(ctx, ri.Name, name, m, oauthConfig, opt) if err != nil { return nil, fmt.Errorf("config failed to refresh token: %w", err) } } err = configExchange(ctx, name, m, oauthConfig, code) if err != nil { return nil, err } return fs.ConfigGoto(newState("*oauth-done")) case "*oauth-done": // Return to the state indicated in the State stack _, returnState := fs.StatePop(stateParams) return fs.ConfigGoto(returnState) } return nil, fmt.Errorf("unknown internal oauth state %q", state) } func init() { // Set the function to avoid circular import fs.ConfigOAuth = ConfigOAuth } // Return true if can run without a webserver and just entering a code func noWebserverNeeded(oauthConfig *oauth2.Config) bool { return oauthConfig.RedirectURL == TitleBarRedirectURL } // get the URL we need to send the user to func getAuthURL(name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) (authURL string, state string, err error) { oauthConfig, _ = overrideCredentials(name, m, oauthConfig) // Make random state state, err = random.Password(128) if err != nil { return "", "", err } // Generate oauth URL opts := opt.OAuth2Opts if !opt.NoOffline { opts = append(opts, oauth2.AccessTypeOffline) } authURL = oauthConfig.AuthCodeURL(state, opts...) return authURL, state, nil } // If TitleBarRedirect is set but we are doing a real oauth, then // override our redirect URL func fixRedirect(oauthConfig *oauth2.Config) *oauth2.Config { switch oauthConfig.RedirectURL { case TitleBarRedirectURL: // copy the config and set to use the internal webserver configCopy := *oauthConfig oauthConfig = &configCopy oauthConfig.RedirectURL = RedirectURL } return oauthConfig } // configSetup does the initial creation of the token // // If opt is nil it will use the default Options. // // It will run an internal webserver to receive the results func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) (string, error) { if opt == nil { opt = &Options{} } authorizeNoAutoBrowserValue, ok := m.Get(config.ConfigAuthNoBrowser) authorizeNoAutoBrowser := ok && authorizeNoAutoBrowserValue != "" authURL, state, err := getAuthURL(name, m, oauthConfig, opt) if err != nil { return "", err } // Prepare webserver server := newAuthServer(opt, bindAddress, state, authURL) err = server.Init() if err != nil { return "", fmt.Errorf("failed to start auth webserver: %w", err) } go server.Serve() defer server.Stop() authURL = "http://" + bindAddress + "/auth?state=" + state if !authorizeNoAutoBrowser { // Open the URL for the user to visit _ = open.Start(authURL) fs.Logf(nil, "If your browser doesn't open automatically go to the following link: %s\n", authURL) } else { fs.Logf(nil, "Please go to the following link: %s\n", authURL) } fs.Logf(nil, "Log in and authorize rclone for access\n") // Read the code via the webserver fs.Logf(nil, "Waiting for code...\n") auth := <-server.result if !auth.OK || auth.Code == "" { return "", auth } fs.Logf(nil, "Got code\n") if opt.CheckAuth != nil { err = opt.CheckAuth(oauthConfig, auth) if err != nil { return "", err } } return auth.Code, nil } // Exchange the code for a token func configExchange(ctx context.Context, name string, m configmap.Mapper, oauthConfig *oauth2.Config, code string) error { ctx = Context(ctx, fshttp.NewClient(ctx)) token, err := oauthConfig.Exchange(ctx, code) if err != nil { return fmt.Errorf("failed to get token: %w", err) } return PutToken(name, m, token, true) } // Local web server for collecting auth type authServer struct { opt *Options state string listener net.Listener bindAddress string authURL string server *http.Server result chan *AuthResult } // newAuthServer makes the webserver for collecting auth func newAuthServer(opt *Options, bindAddress, state, authURL string) *authServer { return &authServer{ opt: opt, state: state, bindAddress: bindAddress, authURL: authURL, // http://host/auth redirects to here result: make(chan *AuthResult, 1), } } // Receive the auth request func (s *authServer) handleAuth(w http.ResponseWriter, req *http.Request) { if req.URL.Path != "/" { fs.Debugf(nil, "Ignoring %s request on auth server to %q", req.Method, req.URL.Path) http.NotFound(w, req) return } fs.Debugf(nil, "Received %s request on auth server to %q", req.Method, req.URL.Path) // Reply with the response to the user and to the channel reply := func(status int, res *AuthResult) { w.WriteHeader(status) w.Header().Set("Content-Type", "text/html") var t = template.Must(template.New("authResponse").Parse(AuthResponseTemplate)) if err := t.Execute(w, res); err != nil { fs.Debugf(nil, "Could not execute template for web response.") } s.result <- res } // Parse the form parameters and save them err := req.ParseForm() if err != nil { reply(http.StatusBadRequest, &AuthResult{ Name: "Parse form error", Description: err.Error(), }) return } // get code, error if empty code := req.Form.Get("code") if code == "" { reply(http.StatusBadRequest, &AuthResult{ Name: "Auth Error", Description: "No code returned by remote server", }) return } // check state state := req.Form.Get("state") if state != s.state && !(state == "" && s.opt.StateBlankOK) { reply(http.StatusBadRequest, &AuthResult{ Name: "Auth state doesn't match", Description: fmt.Sprintf("Expecting %q got %q", s.state, state), }) return } // code OK reply(http.StatusOK, &AuthResult{ OK: true, Code: code, Form: req.Form, }) } // Init gets the internal web server ready to receive config details func (s *authServer) Init() error { fs.Debugf(nil, "Starting auth server on %s", s.bindAddress) mux := http.NewServeMux() s.server = &http.Server{ Addr: s.bindAddress, Handler: mux, } s.server.SetKeepAlivesEnabled(false) mux.HandleFunc("/auth", func(w http.ResponseWriter, req *http.Request) { state := req.FormValue("state") if state != s.state { fs.Debugf(nil, "State did not match: want %q got %q", s.state, state) http.Error(w, "State did not match - please try again", http.StatusForbidden) return } fs.Debugf(nil, "Redirecting browser to: %s", s.authURL) http.Redirect(w, req, s.authURL, http.StatusTemporaryRedirect) }) mux.HandleFunc("/", s.handleAuth) var err error s.listener, err = net.Listen("tcp", s.bindAddress) if err != nil { return err } return nil } // Serve the auth server, doesn't return func (s *authServer) Serve() { err := s.server.Serve(s.listener) fs.Debugf(nil, "Closed auth server with error: %v", err) } // Stop the auth server by closing its socket func (s *authServer) Stop() { fs.Debugf(nil, "Closing auth server") close(s.result) _ = s.listener.Close() // close the server _ = s.server.Close() }
{{ if .Description }}Description: {{ .Description }}
{{ end }} {{ if .Code }}Code: {{ .Code }}
{{ end }} {{ if .HelpURL }}Look here for help: {{ .HelpURL }}
{{ end }} {{ else }} All done. Please go back to rclone. {{ end }}