go-pkg/asynq
2
0
Fork 0
mirror of https://github.com/hibiken/asynq.git synced 2024-12-27 16:13:40 +08:00
Code Issues Projects Releases Wiki Activity

Fix recoverer test

Browse Source
This commit is contained in:
Ken Hibino 2021-06-03 06:58:07 -07:00
parent f9d7af3def
commit 6cce31a134
4 changed files with 78 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
internal/asynqtest/asynqtest.go
View File

@ -123,18 +123,18 @@ func JSON(kv map[string]interface{}) []byte {
} }
// TaskMessageAfterRetry returns an updated copy of t after retry. // TaskMessageAfterRetry returns an updated copy of t after retry.
// It increments retry count and sets the error message. // It increments retry count and sets the error message and last_failed_at time.
func TaskMessageAfterRetry(t base.TaskMessage, errMsg string) *base.TaskMessage { func TaskMessageAfterRetry(t base.TaskMessage, errMsg string, failedAt time.Time) *base.TaskMessage {
t.Retried = t.Retried + 1 t.Retried = t.Retried + 1
t.ErrorMsg = errMsg t.ErrorMsg = errMsg
t.LastFailedAt = time.Now().Unix() // use EquateApproxTime with cmp.Diff t.LastFailedAt = failedAt.Unix()
return &t return &t
} }
// TaskMessageWithError returns an updated copy of t with the given error message. // TaskMessageWithError returns an updated copy of t with the given error message.
func TaskMessageWithError(t base.TaskMessage, errMsg string) *base.TaskMessage { func TaskMessageWithError(t base.TaskMessage, errMsg string, failedAt time.Time) *base.TaskMessage {
t.ErrorMsg = errMsg t.ErrorMsg = errMsg
t.LastFailedAt = time.Now().Unix() // use EquateApproxTime with cmp.Diff t.LastFailedAt = failedAt.Unix()
return &t return &t
} }

81
internal/rdb/rdb_test.go
View File

@ -1078,7 +1078,7 @@ func TestRetry(t *testing.T) {
errMsg string errMsg string
wantActive map[string][]*base.TaskMessage wantActive map[string][]*base.TaskMessage
wantDeadlines map[string][]base.Z wantDeadlines map[string][]base.Z
wantRetry map[string][]base.Z getWantRetry func(failedAt time.Time) map[string][]base.Z
}{ }{
{ {
active: map[string][]*base.TaskMessage{ active: map[string][]*base.TaskMessage{
@ -1099,11 +1099,13 @@ func TestRetry(t *testing.T) {
wantDeadlines: map[string][]base.Z{ wantDeadlines: map[string][]base.Z{
"default": {{Message: t2, Score: t2Deadline}}, "default": {{Message: t2, Score: t2Deadline}},
}, },
wantRetry: map[string][]base.Z{ getWantRetry: func(failedAt time.Time) map[string][]base.Z {
"default": { return map[string][]base.Z{
{Message: h.TaskMessageAfterRetry(*t1, errMsg), Score: now.Add(5 * time.Minute).Unix()}, "default": {
{Message: t3, Score: now.Add(time.Minute).Unix()}, {Message: h.TaskMessageAfterRetry(*t1, errMsg, failedAt), Score: now.Add(5 * time.Minute).Unix()},
}, {Message: t3, Score: now.Add(time.Minute).Unix()},
},
}
}, },
}, },
{ {
@ -1130,11 +1132,13 @@ func TestRetry(t *testing.T) {
"default": {{Message: t1, Score: t1Deadline}, {Message: t2, Score: t2Deadline}}, "default": {{Message: t1, Score: t1Deadline}, {Message: t2, Score: t2Deadline}},
"custom": {}, "custom": {},
}, },
wantRetry: map[string][]base.Z{ getWantRetry: func(failedAt time.Time) map[string][]base.Z {
"default": {}, return map[string][]base.Z{
"custom": { "default": {},
{Message: h.TaskMessageAfterRetry(*t4, errMsg), Score: now.Add(5 * time.Minute).Unix()}, "custom": {
}, {Message: h.TaskMessageAfterRetry(*t4, errMsg, failedAt), Score: now.Add(5 * time.Minute).Unix()},
},
}
}, },
}, },
} }
@ -1145,6 +1149,7 @@ func TestRetry(t *testing.T) {
h.SeedAllDeadlines(t, r.client, tc.deadlines) h.SeedAllDeadlines(t, r.client, tc.deadlines)
h.SeedAllRetryQueues(t, r.client, tc.retry) h.SeedAllRetryQueues(t, r.client, tc.retry)
callTime := time.Now() // time when method was called
err := r.Retry(tc.msg, tc.processAt, tc.errMsg) err := r.Retry(tc.msg, tc.processAt, tc.errMsg)
if err != nil { if err != nil {
t.Errorf("(*RDB).Retry = %v, want nil", err) t.Errorf("(*RDB).Retry = %v, want nil", err)
@ -1167,7 +1172,8 @@ func TestRetry(t *testing.T) {
h.SortZSetEntryOpt, h.SortZSetEntryOpt,
cmpopts.EquateApproxTime(5 * time.Second), // for LastFailedAt field cmpopts.EquateApproxTime(5 * time.Second), // for LastFailedAt field
} }
for queue, want := range tc.wantRetry { wantRetry := tc.getWantRetry(callTime)
for queue, want := range wantRetry {
gotRetry := h.GetRetryEntries(t, r.client, queue) gotRetry := h.GetRetryEntries(t, r.client, queue)
if diff := cmp.Diff(want, gotRetry, cmpOpts...); diff != "" { if diff := cmp.Diff(want, gotRetry, cmpOpts...); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryKey(queue), diff) t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryKey(queue), diff)
@ -1244,13 +1250,13 @@ func TestArchive(t *testing.T) {
// TODO(hibiken): add test cases for trimming // TODO(hibiken): add test cases for trimming
tests := []struct { tests := []struct {
active map[string][]*base.TaskMessage active map[string][]*base.TaskMessage
deadlines map[string][]base.Z deadlines map[string][]base.Z
archived map[string][]base.Z archived map[string][]base.Z
target *base.TaskMessage // task to archive target *base.TaskMessage // task to archive
wantActive map[string][]*base.TaskMessage wantActive map[string][]*base.TaskMessage
wantDeadlines map[string][]base.Z wantDeadlines map[string][]base.Z
wantArchived map[string][]base.Z getWantArchived func(failedAt time.Time) map[string][]base.Z
}{ }{
{ {
active: map[string][]*base.TaskMessage{ active: map[string][]*base.TaskMessage{
@ -1274,11 +1280,13 @@ func TestArchive(t *testing.T) {
wantDeadlines: map[string][]base.Z{ wantDeadlines: map[string][]base.Z{
"default": {{Message: t2, Score: t2Deadline}}, "default": {{Message: t2, Score: t2Deadline}},
}, },
wantArchived: map[string][]base.Z{ getWantArchived: func(failedAt time.Time) map[string][]base.Z {
"default": { return map[string][]base.Z{
{Message: h.TaskMessageWithError(*t1, errMsg), Score: now.Unix()}, "default": {
{Message: t3, Score: now.Add(-time.Hour).Unix()}, {Message: h.TaskMessageWithError(*t1, errMsg, failedAt), Score: failedAt.Unix()},
}, {Message: t3, Score: now.Add(-time.Hour).Unix()},
},
}
}, },
}, },
{ {
@ -1305,10 +1313,12 @@ func TestArchive(t *testing.T) {
{Message: t3, Score: t3Deadline}, {Message: t3, Score: t3Deadline},
}, },
}, },
wantArchived: map[string][]base.Z{ getWantArchived: func(failedAt time.Time) map[string][]base.Z {
"default": { return map[string][]base.Z{
{Message: h.TaskMessageWithError(*t1, errMsg), Score: now.Unix()}, "default": {
}, {Message: h.TaskMessageWithError(*t1, errMsg, failedAt), Score: failedAt.Unix()},
},
}
}, },
}, },
{ {
@ -1337,11 +1347,13 @@ func TestArchive(t *testing.T) {
"default": {{Message: t1, Score: t1Deadline}}, "default": {{Message: t1, Score: t1Deadline}},
"custom": {}, "custom": {},
}, },
wantArchived: map[string][]base.Z{ getWantArchived: func(failedAt time.Time) map[string][]base.Z {
"default": {}, return map[string][]base.Z{
"custom": { "default": {},
{Message: h.TaskMessageWithError(*t4, errMsg), Score: now.Unix()}, "custom": {
}, {Message: h.TaskMessageWithError(*t4, errMsg, failedAt), Score: failedAt.Unix()},
},
}
}, },
}, },
} }
@ -1352,6 +1364,7 @@ func TestArchive(t *testing.T) {
h.SeedAllDeadlines(t, r.client, tc.deadlines) h.SeedAllDeadlines(t, r.client, tc.deadlines)
h.SeedAllArchivedQueues(t, r.client, tc.archived) h.SeedAllArchivedQueues(t, r.client, tc.archived)
callTime := time.Now() // record time `Archive` was called
err := r.Archive(tc.target, errMsg) err := r.Archive(tc.target, errMsg)
if err != nil { if err != nil {
t.Errorf("(*RDB).Archive(%v, %v) = %v, want nil", tc.target, errMsg, err) t.Errorf("(*RDB).Archive(%v, %v) = %v, want nil", tc.target, errMsg, err)
@ -1370,7 +1383,7 @@ func TestArchive(t *testing.T) {
t.Errorf("mismatch found in %q after calling (*RDB).Archive: (-want, +got):\n%s", base.DeadlinesKey(queue), diff) t.Errorf("mismatch found in %q after calling (*RDB).Archive: (-want, +got):\n%s", base.DeadlinesKey(queue), diff)
} }
} }
for queue, want := range tc.wantArchived { for queue, want := range tc.getWantArchived(callTime) {
gotArchived := h.GetArchivedEntries(t, r.client, queue) gotArchived := h.GetArchivedEntries(t, r.client, queue)
if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt, timeCmpOpt); diff != "" { if diff := cmp.Diff(want, gotArchived, h.SortZSetEntryOpt, zScoreCmpOpt, timeCmpOpt); diff != "" {
t.Errorf("mismatch found in %q after calling (*RDB).Archive: (-want, +got):\n%s", base.ArchivedKey(queue), diff) t.Errorf("mismatch found in %q after calling (*RDB).Archive: (-want, +got):\n%s", base.ArchivedKey(queue), diff)

12
processor_test.go
View File

@ -408,8 +408,8 @@ func TestProcessorRetry(t *testing.T) {
p.handler = tc.handler p.handler = tc.handler
p.start(&sync.WaitGroup{}) p.start(&sync.WaitGroup{})
now := time.Now() // time when processor is running runTime := time.Now() // time when processor is running
time.Sleep(tc.wait) // FIXME: This makes test flaky. time.Sleep(tc.wait) // FIXME: This makes test flaky.
p.shutdown() p.shutdown()
cmpOpt := h.EquateInt64Approx(int64(tc.wait.Seconds())) // allow up to a wait-second difference in zset score cmpOpt := h.EquateInt64Approx(int64(tc.wait.Seconds())) // allow up to a wait-second difference in zset score
@ -418,8 +418,8 @@ func TestProcessorRetry(t *testing.T) {
for _, msg := range tc.wantRetry { for _, msg := range tc.wantRetry {
wantRetry = append(wantRetry, wantRetry = append(wantRetry,
base.Z{ base.Z{
Message: h.TaskMessageAfterRetry(*msg, tc.wantErrMsg), Message: h.TaskMessageAfterRetry(*msg, tc.wantErrMsg, runTime),
Score: now.Add(tc.delay).Unix(), Score: runTime.Add(tc.delay).Unix(),
}) })
} }
if diff := cmp.Diff(wantRetry, gotRetry, h.SortZSetEntryOpt, cmpOpt); diff != "" { if diff := cmp.Diff(wantRetry, gotRetry, h.SortZSetEntryOpt, cmpOpt); diff != "" {
@ -431,8 +431,8 @@ func TestProcessorRetry(t *testing.T) {
for _, msg := range tc.wantArchived { for _, msg := range tc.wantArchived {
wantArchived = append(wantArchived, wantArchived = append(wantArchived,
base.Z{ base.Z{
Message: h.TaskMessageWithError(*msg, tc.wantErrMsg), Message: h.TaskMessageWithError(*msg, tc.wantErrMsg, runTime),
Score: now.Unix(), Score: runTime.Unix(),
}) })
} }
if diff := cmp.Diff(wantArchived, gotArchived, h.SortZSetEntryOpt, cmpOpt); diff != "" { if diff := cmp.Diff(wantArchived, gotArchived, h.SortZSetEntryOpt, cmpOpt); diff != "" {

30
recoverer_test.go
View File

@ -64,7 +64,7 @@ func TestRecoverer(t *testing.T) {
"default": {}, "default": {},
}, },
wantRetry: map[string][]*base.TaskMessage{ wantRetry: map[string][]*base.TaskMessage{
"default": {h.TaskMessageAfterRetry(*t1, "deadline exceeded")}, "default": {t1},
}, },
wantArchived: map[string][]*base.TaskMessage{ wantArchived: map[string][]*base.TaskMessage{
"default": {}, "default": {},
@ -101,7 +101,7 @@ func TestRecoverer(t *testing.T) {
"critical": {}, "critical": {},
}, },
wantArchived: map[string][]*base.TaskMessage{ wantArchived: map[string][]*base.TaskMessage{
"default": {h.TaskMessageWithError(*t4, "deadline exceeded")}, "default": {t4},
"critical": {}, "critical": {},
}, },
}, },
@ -137,7 +137,7 @@ func TestRecoverer(t *testing.T) {
"critical": {{Message: t3, Score: oneHourFromNow.Unix()}}, "critical": {{Message: t3, Score: oneHourFromNow.Unix()}},
}, },
wantRetry: map[string][]*base.TaskMessage{ wantRetry: map[string][]*base.TaskMessage{
"default": {h.TaskMessageAfterRetry(*t1, "deadline exceeded")}, "default": {t1},
"critical": {}, "critical": {},
}, },
wantArchived: map[string][]*base.TaskMessage{ wantArchived: map[string][]*base.TaskMessage{
@ -176,8 +176,8 @@ func TestRecoverer(t *testing.T) {
"default": {{Message: t2, Score: oneHourFromNow.Unix()}}, "default": {{Message: t2, Score: oneHourFromNow.Unix()}},
}, },
wantRetry: map[string][]*base.TaskMessage{ wantRetry: map[string][]*base.TaskMessage{
"default": {h.TaskMessageAfterRetry(*t1, "deadline exceeded")}, "default": {t1},
"critical": {h.TaskMessageAfterRetry(*t3, "deadline exceeded")}, "critical": {t3},
}, },
wantArchived: map[string][]*base.TaskMessage{ wantArchived: map[string][]*base.TaskMessage{
"default": {}, "default": {},
@ -238,6 +238,7 @@ func TestRecoverer(t *testing.T) {
var wg sync.WaitGroup var wg sync.WaitGroup
recoverer.start(&wg) recoverer.start(&wg)
runTime := time.Now() // time when recoverer is running
time.Sleep(2 * time.Second) time.Sleep(2 * time.Second)
recoverer.shutdown() recoverer.shutdown()
@ -253,15 +254,24 @@ func TestRecoverer(t *testing.T) {
t.Errorf("%s; mismatch found in %q; (-want,+got)\n%s", tc.desc, base.DeadlinesKey(qname), diff) t.Errorf("%s; mismatch found in %q; (-want,+got)\n%s", tc.desc, base.DeadlinesKey(qname), diff)
} }
} }
for qname, want := range tc.wantRetry { cmpOpt := h.EquateInt64Approx(2) // allow up to two-second difference in `LastFailedAt`
for qname, msgs := range tc.wantRetry {
gotRetry := h.GetRetryMessages(t, r, qname) gotRetry := h.GetRetryMessages(t, r, qname)
if diff := cmp.Diff(want, gotRetry, h.SortMsgOpt); diff != "" { var wantRetry []*base.TaskMessage // Note: construct message here since `LastFailedAt` is relative to each test run
for _, msg := range msgs {
wantRetry = append(wantRetry, h.TaskMessageAfterRetry(*msg, "deadline exceeded", runTime))
}
if diff := cmp.Diff(wantRetry, gotRetry, h.SortMsgOpt, cmpOpt); diff != "" {
t.Errorf("%s; mismatch found in %q: (-want, +got)\n%s", tc.desc, base.RetryKey(qname), diff) t.Errorf("%s; mismatch found in %q: (-want, +got)\n%s", tc.desc, base.RetryKey(qname), diff)
} }
} }
for qname, want := range tc.wantArchived { for qname, msgs := range tc.wantArchived {
gotDead := h.GetArchivedMessages(t, r, qname) gotArchived := h.GetArchivedMessages(t, r, qname)
if diff := cmp.Diff(want, gotDead, h.SortMsgOpt); diff != "" { var wantArchived []*base.TaskMessage
for _, msg := range msgs {
wantArchived = append(wantArchived, h.TaskMessageWithError(*msg, "deadline exceeded", runTime))
}
if diff := cmp.Diff(wantArchived, gotArchived, h.SortMsgOpt, cmpOpt); diff != "" {
t.Errorf("%s; mismatch found in %q: (-want, +got)\n%s", tc.desc, base.ArchivedKey(qname), diff) t.Errorf("%s; mismatch found in %q: (-want, +got)\n%s", tc.desc, base.ArchivedKey(qname), diff)
} }
} }