rpc: treat protocol handshake errors as permanent

treat handshake errors as permanent on the client

The issue was observed by 100% CPU usage due to lack ofrate-limiting in
dataconn.ReqPing retries=> safeguard that
This commit is contained in:
Christian Schwarz
2019-03-15 15:59:47 +01:00
parent 7584c66bdb
commit ab3e783168
4 changed files with 20 additions and 6 deletions

View File

@ -142,6 +142,11 @@ func (c *Client) WaitForConnectivity(ctx context.Context) error {
// dataClient uses transport.Connecter, which doesn't expose FailFast(false)
// => we need to mask dial timeouts
if err, ok := dataErr.(interface{ Temporary() bool }); ok && err.Temporary() {
// Rate-limit pings here in case Temporary() is a mis-classification
// or returns immediately (this is a tight loop in that case)
// TODO keep this in lockstep with controlClient
// => don't use FailFast for control, but check that both control and data worked
time.Sleep(envconst.Duration("ZREPL_RPC_DATACONN_PING_SLEEP", 1*time.Second))
continue
}
// it's not a dial timeout,