mirror of
https://github.com/zrepl/zrepl.git
synced 2025-06-22 18:51:30 +02:00
replication: simpler PermanentError state + handle context cancellation
This commit is contained in:
parent
814fec60f0
commit
4ede99b08c
@ -275,7 +275,7 @@ func (t *tui) renderReplicationReport(rep *replication.Report) {
|
|||||||
t.newline()
|
t.newline()
|
||||||
}
|
}
|
||||||
if rep.SleepUntil.After(time.Now()) &&
|
if rep.SleepUntil.After(time.Now()) &&
|
||||||
state & ^(replication.ContextDone|replication.Completed) != 0 {
|
state & ^(replication.PermanentError|replication.Completed) != 0 {
|
||||||
t.printf("Sleeping until %s (%s left)\n", rep.SleepUntil, rep.SleepUntil.Sub(time.Now()))
|
t.printf("Sleeping until %s (%s left)\n", rep.SleepUntil, rep.SleepUntil.Sub(time.Now()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -9,6 +9,7 @@ import (
|
|||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||||
"math/bits"
|
"math/bits"
|
||||||
|
"net"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -27,7 +28,7 @@ const (
|
|||||||
Working
|
Working
|
||||||
WorkingWait
|
WorkingWait
|
||||||
Completed
|
Completed
|
||||||
ContextDone
|
PermanentError
|
||||||
)
|
)
|
||||||
|
|
||||||
func (s State) rsf() state {
|
func (s State) rsf() state {
|
||||||
@ -65,11 +66,8 @@ type Replication struct {
|
|||||||
completed []*fsrep.Replication
|
completed []*fsrep.Replication
|
||||||
active *ReplicationQueueItemHandle
|
active *ReplicationQueueItemHandle
|
||||||
|
|
||||||
// PlanningError
|
// for PlanningError, WorkingWait and ContextError and Completed
|
||||||
planningError error
|
err error
|
||||||
|
|
||||||
// ContextDone
|
|
||||||
contextError error
|
|
||||||
|
|
||||||
// PlanningError, WorkingWait
|
// PlanningError, WorkingWait
|
||||||
sleepUntil time.Time
|
sleepUntil time.Time
|
||||||
@ -189,6 +187,17 @@ func resolveConflict(conflict error) (path []*pdu.FilesystemVersion, msg string)
|
|||||||
|
|
||||||
var PlanningRetryInterval = 10 * time.Second // FIXME make constant onfigurable
|
var PlanningRetryInterval = 10 * time.Second // FIXME make constant onfigurable
|
||||||
|
|
||||||
|
func isPermanent(err error) bool {
|
||||||
|
switch err {
|
||||||
|
case context.Canceled: return true
|
||||||
|
case context.DeadlineExceeded: return true
|
||||||
|
}
|
||||||
|
if operr, ok := err.(net.Error); ok {
|
||||||
|
return !operr.Temporary()
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func statePlanning(ctx context.Context, sender Sender, receiver Receiver, u updater) state {
|
func statePlanning(ctx context.Context, sender Sender, receiver Receiver, u updater) state {
|
||||||
|
|
||||||
log := getLogger(ctx)
|
log := getLogger(ctx)
|
||||||
@ -196,11 +205,14 @@ func statePlanning(ctx context.Context, sender Sender, receiver Receiver, u upda
|
|||||||
log.Info("start planning")
|
log.Info("start planning")
|
||||||
|
|
||||||
handlePlanningError := func(err error) state {
|
handlePlanningError := func(err error) state {
|
||||||
// FIXME classify error as temporary or permanent / max retry counter
|
|
||||||
return u(func(r *Replication) {
|
return u(func(r *Replication) {
|
||||||
r.sleepUntil = time.Now().Add(PlanningRetryInterval)
|
r.err = err
|
||||||
r.planningError = err
|
if isPermanent(err) {
|
||||||
r.state = PlanningError
|
r.state = PermanentError
|
||||||
|
} else {
|
||||||
|
r.sleepUntil = time.Now().Add(PlanningRetryInterval)
|
||||||
|
r.state = PlanningError
|
||||||
|
}
|
||||||
}).rsf()
|
}).rsf()
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -301,7 +313,7 @@ func statePlanning(ctx context.Context, sender Sender, receiver Receiver, u upda
|
|||||||
return u(func(r *Replication) {
|
return u(func(r *Replication) {
|
||||||
r.completed = nil
|
r.completed = nil
|
||||||
r.queue = q
|
r.queue = q
|
||||||
r.planningError = nil
|
r.err = nil
|
||||||
r.state = Working
|
r.state = Working
|
||||||
}).rsf()
|
}).rsf()
|
||||||
}
|
}
|
||||||
@ -317,8 +329,8 @@ func statePlanningError(ctx context.Context, sender Sender, receiver Receiver, u
|
|||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return u(func(r *Replication) {
|
return u(func(r *Replication) {
|
||||||
r.state = ContextDone
|
r.state = PermanentError
|
||||||
r.contextError = ctx.Err()
|
r.err = ctx.Err()
|
||||||
}).rsf()
|
}).rsf()
|
||||||
case <-t.C:
|
case <-t.C:
|
||||||
case <-wakeup.Wait(ctx):
|
case <-wakeup.Wait(ctx):
|
||||||
@ -358,6 +370,17 @@ func stateWorking(ctx context.Context, sender Sender, receiver Receiver, u updat
|
|||||||
active.Update(state, nextStepDate, retryWaitUntil)
|
active.Update(state, nextStepDate, retryWaitUntil)
|
||||||
r.active = nil
|
r.active = nil
|
||||||
}).rsf()
|
}).rsf()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return u(func(r *Replication) {
|
||||||
|
r.err = ctx.Err()
|
||||||
|
r.state = PermanentError
|
||||||
|
}).rsf()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
return u(nil).rsf()
|
||||||
}
|
}
|
||||||
|
|
||||||
func stateWorkingWait(ctx context.Context, sender Sender, receiver Receiver, u updater) state {
|
func stateWorkingWait(ctx context.Context, sender Sender, receiver Receiver, u updater) state {
|
||||||
@ -371,8 +394,8 @@ func stateWorkingWait(ctx context.Context, sender Sender, receiver Receiver, u u
|
|||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return u(func(r *Replication) {
|
return u(func(r *Replication) {
|
||||||
r.state = ContextDone
|
r.state = PermanentError
|
||||||
r.contextError = ctx.Err()
|
r.err = ctx.Err()
|
||||||
}).rsf()
|
}).rsf()
|
||||||
|
|
||||||
case <-t.C:
|
case <-t.C:
|
||||||
@ -395,12 +418,9 @@ func (r *Replication) Report() *Report {
|
|||||||
SleepUntil: r.sleepUntil,
|
SleepUntil: r.sleepUntil,
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.state&(Planning|PlanningError|ContextDone) != 0 {
|
if r.state&(Planning|PlanningError|PermanentError) != 0 {
|
||||||
switch r.state {
|
if r.err != nil {
|
||||||
case PlanningError:
|
rep.Problem = r.err.Error()
|
||||||
rep.Problem = r.planningError.Error()
|
|
||||||
case ContextDone:
|
|
||||||
rep.Problem = r.contextError.Error()
|
|
||||||
}
|
}
|
||||||
return &rep
|
return &rep
|
||||||
}
|
}
|
||||||
|
@ -11,7 +11,7 @@ const (
|
|||||||
_StateName_1 = "Working"
|
_StateName_1 = "Working"
|
||||||
_StateName_2 = "WorkingWait"
|
_StateName_2 = "WorkingWait"
|
||||||
_StateName_3 = "Completed"
|
_StateName_3 = "Completed"
|
||||||
_StateName_4 = "ContextDone"
|
_StateName_4 = "PermanentError"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -19,7 +19,7 @@ var (
|
|||||||
_StateIndex_1 = [...]uint8{0, 7}
|
_StateIndex_1 = [...]uint8{0, 7}
|
||||||
_StateIndex_2 = [...]uint8{0, 11}
|
_StateIndex_2 = [...]uint8{0, 11}
|
||||||
_StateIndex_3 = [...]uint8{0, 9}
|
_StateIndex_3 = [...]uint8{0, 9}
|
||||||
_StateIndex_4 = [...]uint8{0, 11}
|
_StateIndex_4 = [...]uint8{0, 14}
|
||||||
)
|
)
|
||||||
|
|
||||||
func (i State) String() string {
|
func (i State) String() string {
|
||||||
@ -48,7 +48,7 @@ var _StateNameToValueMap = map[string]State{
|
|||||||
_StateName_1[0:7]: 4,
|
_StateName_1[0:7]: 4,
|
||||||
_StateName_2[0:11]: 8,
|
_StateName_2[0:11]: 8,
|
||||||
_StateName_3[0:9]: 16,
|
_StateName_3[0:9]: 16,
|
||||||
_StateName_4[0:11]: 32,
|
_StateName_4[0:14]: 32,
|
||||||
}
|
}
|
||||||
|
|
||||||
// StateString retrieves an enum value from the enum constants string name.
|
// StateString retrieves an enum value from the enum constants string name.
|
||||||
|
Loading…
x
Reference in New Issue
Block a user