go-pkg/asynq
2
0
Fork 0
mirror of https://github.com/hibiken/asynq.git synced 2025-08-19 15:08:55 +08:00
Code Issues Projects Releases Wiki Activity

Create base internal package

Browse Source
This commit is contained in:
Ken Hibino
2019-12-22 07:15:45 -08:00
parent 5de314400d
commit 3fd248615b
12 changed files with 390 additions and 387 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/rdb/helpers_test.go
View File

@@ -9,6 +9,7 @@ import (
"github.com/go-redis/redis/v7"
"github.com/google/go-cmp/cmp"
"github.com/hibiken/asynq/internal/base"
"github.com/rs/xid"
)
@@ -37,8 +38,8 @@ func flushDB(t *testing.T, r *RDB) {
}
}
var sortMsgOpt = cmp.Transformer("SortMsg", func(in []*TaskMessage) []*TaskMessage {
out := append([]*TaskMessage(nil), in...) // Copy input to avoid mutating it
var sortMsgOpt = cmp.Transformer("SortMsg", func(in []*base.TaskMessage) []*base.TaskMessage {
out := append([]*base.TaskMessage(nil), in...) // Copy input to avoid mutating it
sort.Slice(out, func(i, j int) bool {
return out[i].ID.String() < out[j].ID.String()
})
@@ -53,8 +54,8 @@ var sortZSetEntryOpt = cmp.Transformer("SortZSetEntry", func(in []sortedSetEntry
return out
})
func newTaskMessage(taskType string, payload map[string]interface{}) *TaskMessage {
return &TaskMessage{
func newTaskMessage(taskType string, payload map[string]interface{}) *base.TaskMessage {
return &base.TaskMessage{
ID: xid.New(),
Type: taskType,
Queue: "default",
@@ -63,7 +64,7 @@ func newTaskMessage(taskType string, payload map[string]interface{}) *TaskMessag
}
}
func mustMarshal(t *testing.T, msg *TaskMessage) string {
func mustMarshal(t *testing.T, msg *base.TaskMessage) string {
t.Helper()
data, err := json.Marshal(msg)
if err != nil {
@@ -72,9 +73,9 @@ func mustMarshal(t *testing.T, msg *TaskMessage) string {
return string(data)
}
func mustUnmarshal(t *testing.T, data string) *TaskMessage {
func mustUnmarshal(t *testing.T, data string) *base.TaskMessage {
t.Helper()
var msg TaskMessage
var msg base.TaskMessage
err := json.Unmarshal([]byte(data), &msg)
if err != nil {
t.Fatal(err)
@@ -82,7 +83,7 @@ func mustUnmarshal(t *testing.T, data string) *TaskMessage {
return &msg
}
func mustMarshalSlice(t *testing.T, msgs []*TaskMessage) []string {
func mustMarshalSlice(t *testing.T, msgs []*base.TaskMessage) []string {
t.Helper()
var data []string
for _, m := range msgs {
@@ -91,16 +92,16 @@ func mustMarshalSlice(t *testing.T, msgs []*TaskMessage) []string {
return data
}
func mustUnmarshalSlice(t *testing.T, data []string) []*TaskMessage {
func mustUnmarshalSlice(t *testing.T, data []string) []*base.TaskMessage {
t.Helper()
var msgs []*TaskMessage
var msgs []*base.TaskMessage
for _, s := range data {
msgs = append(msgs, mustUnmarshal(t, s))
}
return msgs
}
func seedRedisList(t *testing.T, c *redis.Client, key string, msgs []*TaskMessage) {
func seedRedisList(t *testing.T, c *redis.Client, key string, msgs []*base.TaskMessage) {
data := mustMarshalSlice(t, msgs)
for _, s := range data {
if err := c.LPush(key, s).Err(); err != nil {
@@ -120,36 +121,36 @@ func seedRedisZSet(t *testing.T, c *redis.Client, key string, items []sortedSetE
// scheduledEntry represents an item in redis sorted set (aka ZSET).
type sortedSetEntry struct {
msg *TaskMessage
msg *base.TaskMessage
score int64
}
// seedDefaultQueue initializes the default queue with the given messages.
func seedDefaultQueue(t *testing.T, r *RDB, msgs []*TaskMessage) {
func seedDefaultQueue(t *testing.T, r *RDB, msgs []*base.TaskMessage) {
t.Helper()
seedRedisList(t, r.client, defaultQ, msgs)
seedRedisList(t, r.client, base.DefaultQueue, msgs)
}
// seedInProgressQueue initializes the in-progress queue with the given messages.
func seedInProgressQueue(t *testing.T, r *RDB, msgs []*TaskMessage) {
func seedInProgressQueue(t *testing.T, r *RDB, msgs []*base.TaskMessage) {
t.Helper()
seedRedisList(t, r.client, inProgressQ, msgs)
seedRedisList(t, r.client, base.InProgressQueue, msgs)
}
// seedScheduledQueue initializes the scheduled queue with the given messages.
func seedScheduledQueue(t *testing.T, r *RDB, entries []sortedSetEntry) {
t.Helper()
seedRedisZSet(t, r.client, scheduledQ, entries)
seedRedisZSet(t, r.client, base.ScheduledQueue, entries)
}
// seedRetryQueue initializes the retry queue with the given messages.
func seedRetryQueue(t *testing.T, r *RDB, entries []sortedSetEntry) {
t.Helper()
seedRedisZSet(t, r.client, retryQ, entries)
seedRedisZSet(t, r.client, base.RetryQueue, entries)
}
// seedDeadQueue initializes the dead queue with the given messages.
func seedDeadQueue(t *testing.T, r *RDB, entries []sortedSetEntry) {
t.Helper()
seedRedisZSet(t, r.client, deadQ, entries)
seedRedisZSet(t, r.client, base.DeadQueue, entries)
}

59
internal/rdb/inspect.go
View File

@@ -6,6 +6,7 @@ import (
"time"
"github.com/go-redis/redis/v7"
"github.com/hibiken/asynq/internal/base"
"github.com/rs/xid"
)
@@ -73,11 +74,11 @@ type DeadTask struct {
// CurrentStats returns a current state of the queues.
func (r *RDB) CurrentStats() (*Stats, error) {
pipe := r.client.Pipeline()
qlen := pipe.LLen(defaultQ)
plen := pipe.LLen(inProgressQ)
slen := pipe.ZCard(scheduledQ)
rlen := pipe.ZCard(retryQ)
dlen := pipe.ZCard(deadQ)
qlen := pipe.LLen(base.DefaultQueue)
plen := pipe.LLen(base.InProgressQueue)
slen := pipe.ZCard(base.ScheduledQueue)
rlen := pipe.ZCard(base.RetryQueue)
dlen := pipe.ZCard(base.DeadQueue)
_, err := pipe.Exec()
if err != nil {
return nil, err
@@ -94,13 +95,13 @@ func (r *RDB) CurrentStats() (*Stats, error) {
// ListEnqueued returns all enqueued tasks that are ready to be processed.
func (r *RDB) ListEnqueued() ([]*EnqueuedTask, error) {
data, err := r.client.LRange(defaultQ, 0, -1).Result()
data, err := r.client.LRange(base.DefaultQueue, 0, -1).Result()
if err != nil {
return nil, err
}
var tasks []*EnqueuedTask
for _, s := range data {
var msg TaskMessage
var msg base.TaskMessage
err := json.Unmarshal([]byte(s), &msg)
if err != nil {
// continue // bad data, ignore and continue
@@ -117,13 +118,13 @@ func (r *RDB) ListEnqueued() ([]*EnqueuedTask, error) {
// ListInProgress returns all tasks that are currently being processed.
func (r *RDB) ListInProgress() ([]*InProgressTask, error) {
data, err := r.client.LRange(inProgressQ, 0, -1).Result()
data, err := r.client.LRange(base.InProgressQueue, 0, -1).Result()
if err != nil {
return nil, err
}
var tasks []*InProgressTask
for _, s := range data {
var msg TaskMessage
var msg base.TaskMessage
err := json.Unmarshal([]byte(s), &msg)
if err != nil {
continue // bad data, ignore and continue
@@ -140,7 +141,7 @@ func (r *RDB) ListInProgress() ([]*InProgressTask, error) {
// ListScheduled returns all tasks that are scheduled to be processed
// in the future.
func (r *RDB) ListScheduled() ([]*ScheduledTask, error) {
data, err := r.client.ZRangeWithScores(scheduledQ, 0, -1).Result()
data, err := r.client.ZRangeWithScores(base.ScheduledQueue, 0, -1).Result()
if err != nil {
return nil, err
}
@@ -150,7 +151,7 @@ func (r *RDB) ListScheduled() ([]*ScheduledTask, error) {
if !ok {
continue // bad data, ignore and continue
}
var msg TaskMessage
var msg base.TaskMessage
err := json.Unmarshal([]byte(s), &msg)
if err != nil {
continue // bad data, ignore and continue
@@ -170,7 +171,7 @@ func (r *RDB) ListScheduled() ([]*ScheduledTask, error) {
// ListRetry returns all tasks that have failed before and willl be retried
// in the future.
func (r *RDB) ListRetry() ([]*RetryTask, error) {
data, err := r.client.ZRangeWithScores(retryQ, 0, -1).Result()
data, err := r.client.ZRangeWithScores(base.RetryQueue, 0, -1).Result()
if err != nil {
return nil, err
}
@@ -180,7 +181,7 @@ func (r *RDB) ListRetry() ([]*RetryTask, error) {
if !ok {
continue // bad data, ignore and continue
}
var msg TaskMessage
var msg base.TaskMessage
err := json.Unmarshal([]byte(s), &msg)
if err != nil {
continue // bad data, ignore and continue
@@ -202,7 +203,7 @@ func (r *RDB) ListRetry() ([]*RetryTask, error) {
// ListDead returns all tasks that have exhausted its retry limit.
func (r *RDB) ListDead() ([]*DeadTask, error) {
data, err := r.client.ZRangeWithScores(deadQ, 0, -1).Result()
data, err := r.client.ZRangeWithScores(base.DeadQueue, 0, -1).Result()
if err != nil {
return nil, err
}
@@ -212,7 +213,7 @@ func (r *RDB) ListDead() ([]*DeadTask, error) {
if !ok {
continue // bad data, ignore and continue
}
var msg TaskMessage
var msg base.TaskMessage
err := json.Unmarshal([]byte(s), &msg)
if err != nil {
continue // bad data, ignore and continue
@@ -234,7 +235,7 @@ func (r *RDB) ListDead() ([]*DeadTask, error) {
// and enqueues it for processing. If a task that matches the id and score
// does not exist, it returns ErrTaskNotFound.
func (r *RDB) EnqueueDeadTask(id xid.ID, score int64) error {
n, err := r.removeAndEnqueue(deadQ, id.String(), float64(score))
n, err := r.removeAndEnqueue(base.DeadQueue, id.String(), float64(score))
if err != nil {
return err
}
@@ -248,7 +249,7 @@ func (r *RDB) EnqueueDeadTask(id xid.ID, score int64) error {
// and enqueues it for processing. If a task that matches the id and score
// does not exist, it returns ErrTaskNotFound.
func (r *RDB) EnqueueRetryTask(id xid.ID, score int64) error {
n, err := r.removeAndEnqueue(retryQ, id.String(), float64(score))
n, err := r.removeAndEnqueue(base.RetryQueue, id.String(), float64(score))
if err != nil {
return err
}
@@ -262,7 +263,7 @@ func (r *RDB) EnqueueRetryTask(id xid.ID, score int64) error {
// and enqueues it for processing. If a task that matches the id and score does not
// exist, it returns ErrTaskNotFound.
func (r *RDB) EnqueueScheduledTask(id xid.ID, score int64) error {
n, err := r.removeAndEnqueue(scheduledQ, id.String(), float64(score))
n, err := r.removeAndEnqueue(base.ScheduledQueue, id.String(), float64(score))
if err != nil {
return err
}
@@ -275,19 +276,19 @@ func (r *RDB) EnqueueScheduledTask(id xid.ID, score int64) error {
// EnqueueAllScheduledTasks enqueues all tasks from scheduled queue
// and returns the number of tasks enqueued.
func (r *RDB) EnqueueAllScheduledTasks() (int64, error) {
return r.removeAndEnqueueAll(scheduledQ)
return r.removeAndEnqueueAll(base.ScheduledQueue)
}
// EnqueueAllRetryTasks enqueues all tasks from retry queue
// and returns the number of tasks enqueued.
func (r *RDB) EnqueueAllRetryTasks() (int64, error) {
return r.removeAndEnqueueAll(retryQ)
return r.removeAndEnqueueAll(base.RetryQueue)
}
// EnqueueAllDeadTasks enqueues all tasks from dead queue
// and returns the number of tasks enqueued.
func (r *RDB) EnqueueAllDeadTasks() (int64, error) {
return r.removeAndEnqueueAll(deadQ)
return r.removeAndEnqueueAll(base.DeadQueue)
}
func (r *RDB) removeAndEnqueue(zset, id string, score float64) (int64, error) {
@@ -303,7 +304,7 @@ func (r *RDB) removeAndEnqueue(zset, id string, score float64) (int64, error) {
end
return 0
`)
res, err := script.Run(r.client, []string{zset, defaultQ}, score, id).Result()
res, err := script.Run(r.client, []string{zset, base.DefaultQueue}, score, id).Result()
if err != nil {
return 0, err
}
@@ -323,7 +324,7 @@ func (r *RDB) removeAndEnqueueAll(zset string) (int64, error) {
end
return table.getn(msgs)
`)
res, err := script.Run(r.client, []string{zset, defaultQ}).Result()
res, err := script.Run(r.client, []string{zset, base.DefaultQueue}).Result()
if err != nil {
return 0, err
}
@@ -338,21 +339,21 @@ func (r *RDB) removeAndEnqueueAll(zset string) (int64, error) {
// and deletes it. If a task that matches the id and score does not exist,
// it returns ErrTaskNotFound.
func (r *RDB) DeleteDeadTask(id xid.ID, score int64) error {
return r.deleteTask(deadQ, id.String(), float64(score))
return r.deleteTask(base.DeadQueue, id.String(), float64(score))
}
// DeleteRetryTask finds a task that matches the given id and score from retry queue
// and deletes it. If a task that matches the id and score does not exist,
// it returns ErrTaskNotFound.
func (r *RDB) DeleteRetryTask(id xid.ID, score int64) error {
return r.deleteTask(retryQ, id.String(), float64(score))
return r.deleteTask(base.RetryQueue, id.String(), float64(score))
}
// DeleteScheduledTask finds a task that matches the given id and score from
// scheduled queue and deletes it. If a task that matches the id and score
//does not exist, it returns ErrTaskNotFound.
func (r *RDB) DeleteScheduledTask(id xid.ID, score int64) error {
return r.deleteTask(scheduledQ, id.String(), float64(score))
return r.deleteTask(base.ScheduledQueue, id.String(), float64(score))
}
func (r *RDB) deleteTask(zset, id string, score float64) error {
@@ -383,15 +384,15 @@ func (r *RDB) deleteTask(zset, id string, score float64) error {
// DeleteAllDeadTasks deletes all tasks from the dead queue.
func (r *RDB) DeleteAllDeadTasks() error {
return r.client.Del(deadQ).Err()
return r.client.Del(base.DeadQueue).Err()
}
// DeleteAllRetryTasks deletes all tasks from the dead queue.
func (r *RDB) DeleteAllRetryTasks() error {
return r.client.Del(retryQ).Err()
return r.client.Del(base.RetryQueue).Err()
}
// DeleteAllScheduledTasks deletes all tasks from the dead queue.
func (r *RDB) DeleteAllScheduledTasks() error {
return r.client.Del(scheduledQ).Err()
return r.client.Del(base.ScheduledQueue).Err()
}

179
internal/rdb/inspect_test.go
View File

@@ -6,6 +6,7 @@ import (
"time"
"github.com/google/go-cmp/cmp"
"github.com/hibiken/asynq/internal/base"
"github.com/rs/xid"
)
@@ -55,16 +56,16 @@ func TestCurrentStats(t *testing.T) {
m4 := newTaskMessage("sync", nil)
tests := []struct {
enqueued []*TaskMessage
inProgress []*TaskMessage
enqueued []*base.TaskMessage
inProgress []*base.TaskMessage
scheduled []sortedSetEntry
retry []sortedSetEntry
dead []sortedSetEntry
want *Stats
}{
{
enqueued: []*TaskMessage{m1},
inProgress: []*TaskMessage{m2},
enqueued: []*base.TaskMessage{m1},
inProgress: []*base.TaskMessage{m2},
scheduled: []sortedSetEntry{
{m3, time.Now().Add(time.Hour).Unix()},
{m4, time.Now().Unix()}},
@@ -80,8 +81,8 @@ func TestCurrentStats(t *testing.T) {
},
},
{
enqueued: []*TaskMessage{},
inProgress: []*TaskMessage{},
enqueued: []*base.TaskMessage{},
inProgress: []*base.TaskMessage{},
scheduled: []sortedSetEntry{
{m3, time.Now().Unix()},
{m4, time.Now().Unix()}},
@@ -130,15 +131,15 @@ func TestListEnqueued(t *testing.T) {
t1 := &EnqueuedTask{ID: m1.ID, Type: m1.Type, Payload: m1.Payload}
t2 := &EnqueuedTask{ID: m2.ID, Type: m2.Type, Payload: m2.Payload}
tests := []struct {
enqueued []*TaskMessage
enqueued []*base.TaskMessage
want []*EnqueuedTask
}{
{
enqueued: []*TaskMessage{m1, m2},
enqueued: []*base.TaskMessage{m1, m2},
want: []*EnqueuedTask{t1, t2},
},
{
enqueued: []*TaskMessage{},
enqueued: []*base.TaskMessage{},
want: []*EnqueuedTask{},
},
}
@@ -174,15 +175,15 @@ func TestListInProgress(t *testing.T) {
t1 := &InProgressTask{ID: m1.ID, Type: m1.Type, Payload: m1.Payload}
t2 := &InProgressTask{ID: m2.ID, Type: m2.Type, Payload: m2.Payload}
tests := []struct {
inProgress []*TaskMessage
inProgress []*base.TaskMessage
want []*InProgressTask
}{
{
inProgress: []*TaskMessage{m1, m2},
inProgress: []*base.TaskMessage{m1, m2},
want: []*InProgressTask{t1, t2},
},
{
inProgress: []*TaskMessage{},
inProgress: []*base.TaskMessage{},
want: []*InProgressTask{},
},
}
@@ -262,7 +263,7 @@ func TestListScheduled(t *testing.T) {
func TestListRetry(t *testing.T) {
r := setup(t)
m1 := &TaskMessage{
m1 := &base.TaskMessage{
ID: xid.New(),
Type: "send_email",
Queue: "default",
@@ -271,7 +272,7 @@ func TestListRetry(t *testing.T) {
Retry: 25,
Retried: 10,
}
m2 := &TaskMessage{
m2 := &base.TaskMessage{
ID: xid.New(),
Type: "reindex",
Queue: "default",
@@ -346,14 +347,14 @@ func TestListRetry(t *testing.T) {
func TestListDead(t *testing.T) {
r := setup(t)
m1 := &TaskMessage{
m1 := &base.TaskMessage{
ID: xid.New(),
Type: "send_email",
Queue: "default",
Payload: map[string]interface{}{"subject": "hello"},
ErrorMsg: "email server not responding",
}
m2 := &TaskMessage{
m2 := &base.TaskMessage{
ID: xid.New(),
Type: "reindex",
Queue: "default",
@@ -434,8 +435,8 @@ func TestEnqueueDeadTask(t *testing.T) {
score int64
id xid.ID
want error // expected return value from calling EnqueueDeadTask
wantDead []*TaskMessage
wantEnqueued []*TaskMessage
wantDead []*base.TaskMessage
wantEnqueued []*base.TaskMessage
}{
{
dead: []sortedSetEntry{
@@ -445,8 +446,8 @@ func TestEnqueueDeadTask(t *testing.T) {
score: s2,
id: t2.ID,
want: nil,
wantDead: []*TaskMessage{t1},
wantEnqueued: []*TaskMessage{t2},
wantDead: []*base.TaskMessage{t1},
wantEnqueued: []*base.TaskMessage{t2},
},
{
dead: []sortedSetEntry{
@@ -456,8 +457,8 @@ func TestEnqueueDeadTask(t *testing.T) {
score: 123,
id: t2.ID,
want: ErrTaskNotFound,
wantDead: []*TaskMessage{t1, t2},
wantEnqueued: []*TaskMessage{},
wantDead: []*base.TaskMessage{t1, t2},
wantEnqueued: []*base.TaskMessage{},
},
}
@@ -470,15 +471,15 @@ func TestEnqueueDeadTask(t *testing.T) {
t.Errorf("r.EnqueueDeadTask(%s, %d) = %v, want %v", tc.id, tc.score, got, tc.want)
continue
}
gotEnqueuedRaw := r.client.LRange(defaultQ, 0, -1).Val()
gotEnqueuedRaw := r.client.LRange(base.DefaultQueue, 0, -1).Val()
gotEnqueued := mustUnmarshalSlice(t, gotEnqueuedRaw)
if diff := cmp.Diff(tc.wantEnqueued, gotEnqueued, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", defaultQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.DefaultQueue, diff)
}
gotDeadRaw := r.client.ZRange(deadQ, 0, -1).Val()
gotDeadRaw := r.client.ZRange(base.DeadQueue, 0, -1).Val()
gotDead := mustUnmarshalSlice(t, gotDeadRaw)
if diff := cmp.Diff(tc.wantDead, gotDead, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q, (-want, +got)\n%s", deadQ, diff)
t.Errorf("mismatch found in %q, (-want, +got)\n%s", base.DeadQueue, diff)
}
}
}
@@ -495,8 +496,8 @@ func TestEnqueueRetryTask(t *testing.T) {
score int64
id xid.ID
want error // expected return value from calling EnqueueRetryTask
wantRetry []*TaskMessage
wantEnqueued []*TaskMessage
wantRetry []*base.TaskMessage
wantEnqueued []*base.TaskMessage
}{
{
retry: []sortedSetEntry{
@@ -506,8 +507,8 @@ func TestEnqueueRetryTask(t *testing.T) {
score: s2,
id: t2.ID,
want: nil,
wantRetry: []*TaskMessage{t1},
wantEnqueued: []*TaskMessage{t2},
wantRetry: []*base.TaskMessage{t1},
wantEnqueued: []*base.TaskMessage{t2},
},
{
retry: []sortedSetEntry{
@@ -517,8 +518,8 @@ func TestEnqueueRetryTask(t *testing.T) {
score: 123,
id: t2.ID,
want: ErrTaskNotFound,
wantRetry: []*TaskMessage{t1, t2},
wantEnqueued: []*TaskMessage{},
wantRetry: []*base.TaskMessage{t1, t2},
wantEnqueued: []*base.TaskMessage{},
},
}
@@ -532,15 +533,15 @@ func TestEnqueueRetryTask(t *testing.T) {
t.Errorf("r.EnqueueRetryTask(%s, %d) = %v, want %v", tc.id, tc.score, got, tc.want)
continue
}
gotEnqueuedRaw := r.client.LRange(defaultQ, 0, -1).Val()
gotEnqueuedRaw := r.client.LRange(base.DefaultQueue, 0, -1).Val()
gotEnqueued := mustUnmarshalSlice(t, gotEnqueuedRaw)
if diff := cmp.Diff(tc.wantEnqueued, gotEnqueued, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", defaultQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.DefaultQueue, diff)
}
gotRetryRaw := r.client.ZRange(retryQ, 0, -1).Val()
gotRetryRaw := r.client.ZRange(base.RetryQueue, 0, -1).Val()
gotRetry := mustUnmarshalSlice(t, gotRetryRaw)
if diff := cmp.Diff(tc.wantRetry, gotRetry, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q, (-want, +got)\n%s", retryQ, diff)
t.Errorf("mismatch found in %q, (-want, +got)\n%s", base.RetryQueue, diff)
}
}
}
@@ -557,8 +558,8 @@ func TestEnqueueScheduledTask(t *testing.T) {
score int64
id xid.ID
want error // expected return value from calling EnqueueScheduledTask
wantScheduled []*TaskMessage
wantEnqueued []*TaskMessage
wantScheduled []*base.TaskMessage
wantEnqueued []*base.TaskMessage
}{
{
scheduled: []sortedSetEntry{
@@ -568,8 +569,8 @@ func TestEnqueueScheduledTask(t *testing.T) {
score: s2,
id: t2.ID,
want: nil,
wantScheduled: []*TaskMessage{t1},
wantEnqueued: []*TaskMessage{t2},
wantScheduled: []*base.TaskMessage{t1},
wantEnqueued: []*base.TaskMessage{t2},
},
{
scheduled: []sortedSetEntry{
@@ -579,8 +580,8 @@ func TestEnqueueScheduledTask(t *testing.T) {
score: 123,
id: t2.ID,
want: ErrTaskNotFound,
wantScheduled: []*TaskMessage{t1, t2},
wantEnqueued: []*TaskMessage{},
wantScheduled: []*base.TaskMessage{t1, t2},
wantEnqueued: []*base.TaskMessage{},
},
}
@@ -593,15 +594,15 @@ func TestEnqueueScheduledTask(t *testing.T) {
t.Errorf("r.EnqueueRetryTask(%s, %d) = %v, want %v", tc.id, tc.score, got, tc.want)
continue
}
gotEnqueuedRaw := r.client.LRange(defaultQ, 0, -1).Val()
gotEnqueuedRaw := r.client.LRange(base.DefaultQueue, 0, -1).Val()
gotEnqueued := mustUnmarshalSlice(t, gotEnqueuedRaw)
if diff := cmp.Diff(tc.wantEnqueued, gotEnqueued, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", defaultQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.DefaultQueue, diff)
}
gotScheduledRaw := r.client.ZRange(scheduledQ, 0, -1).Val()
gotScheduledRaw := r.client.ZRange(base.ScheduledQueue, 0, -1).Val()
gotScheduled := mustUnmarshalSlice(t, gotScheduledRaw)
if diff := cmp.Diff(tc.wantScheduled, gotScheduled, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q, (-want, +got)\n%s", scheduledQ, diff)
t.Errorf("mismatch found in %q, (-want, +got)\n%s", base.ScheduledQueue, diff)
}
}
}
@@ -616,7 +617,7 @@ func TestEnqueueAllScheduledTasks(t *testing.T) {
desc string
scheduled []sortedSetEntry
want int64
wantEnqueued []*TaskMessage
wantEnqueued []*base.TaskMessage
}{
{
desc: "with tasks in scheduled queue",
@@ -626,13 +627,13 @@ func TestEnqueueAllScheduledTasks(t *testing.T) {
{t3, time.Now().Add(time.Hour).Unix()},
},
want: 3,
wantEnqueued: []*TaskMessage{t1, t2, t3},
wantEnqueued: []*base.TaskMessage{t1, t2, t3},
},
{
desc: "with empty scheduled queue",
scheduled: []sortedSetEntry{},
want: 0,
wantEnqueued: []*TaskMessage{},
wantEnqueued: []*base.TaskMessage{},
},
}
@@ -652,10 +653,10 @@ func TestEnqueueAllScheduledTasks(t *testing.T) {
tc.desc, got, err, tc.want)
}
gotEnqueuedRaw := r.client.LRange(defaultQ, 0, -1).Val()
gotEnqueuedRaw := r.client.LRange(base.DefaultQueue, 0, -1).Val()
gotEnqueued := mustUnmarshalSlice(t, gotEnqueuedRaw)
if diff := cmp.Diff(tc.wantEnqueued, gotEnqueued, sortMsgOpt); diff != "" {
t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.desc, defaultQ, diff)
t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.desc, base.DefaultQueue, diff)
}
}
}
@@ -670,7 +671,7 @@ func TestEnqueueAllRetryTasks(t *testing.T) {
description string
retry []sortedSetEntry
want int64
wantEnqueued []*TaskMessage
wantEnqueued []*base.TaskMessage
}{
{
description: "with tasks in retry queue",
@@ -680,13 +681,13 @@ func TestEnqueueAllRetryTasks(t *testing.T) {
{t3, time.Now().Add(time.Hour).Unix()},
},
want: 3,
wantEnqueued: []*TaskMessage{t1, t2, t3},
wantEnqueued: []*base.TaskMessage{t1, t2, t3},
},
{
description: "with empty retry queue",
retry: []sortedSetEntry{},
want: 0,
wantEnqueued: []*TaskMessage{},
wantEnqueued: []*base.TaskMessage{},
},
}
@@ -706,10 +707,10 @@ func TestEnqueueAllRetryTasks(t *testing.T) {
tc.description, got, err, tc.want)
}
gotEnqueuedRaw := r.client.LRange(defaultQ, 0, -1).Val()
gotEnqueuedRaw := r.client.LRange(base.DefaultQueue, 0, -1).Val()
gotEnqueued := mustUnmarshalSlice(t, gotEnqueuedRaw)
if diff := cmp.Diff(tc.wantEnqueued, gotEnqueued, sortMsgOpt); diff != "" {
t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.description, defaultQ, diff)
t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.description, base.DefaultQueue, diff)
}
}
}
@@ -724,7 +725,7 @@ func TestEnqueueAllDeadTasks(t *testing.T) {
desc string
dead []sortedSetEntry
want int64
wantEnqueued []*TaskMessage
wantEnqueued []*base.TaskMessage
}{
{
desc: "with tasks in dead queue",
@@ -734,13 +735,13 @@ func TestEnqueueAllDeadTasks(t *testing.T) {
{t3, time.Now().Add(-time.Minute).Unix()},
},
want: 3,
wantEnqueued: []*TaskMessage{t1, t2, t3},
wantEnqueued: []*base.TaskMessage{t1, t2, t3},
},
{
desc: "with empty dead queue",
dead: []sortedSetEntry{},
want: 0,
wantEnqueued: []*TaskMessage{},
wantEnqueued: []*base.TaskMessage{},
},
}
@@ -760,10 +761,10 @@ func TestEnqueueAllDeadTasks(t *testing.T) {
tc.desc, got, err, tc.want)
}
gotEnqueuedRaw := r.client.LRange(defaultQ, 0, -1).Val()
gotEnqueuedRaw := r.client.LRange(base.DefaultQueue, 0, -1).Val()
gotEnqueued := mustUnmarshalSlice(t, gotEnqueuedRaw)
if diff := cmp.Diff(tc.wantEnqueued, gotEnqueued, sortMsgOpt); diff != "" {
t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.desc, defaultQ, diff)
t.Errorf("%s; mismatch found in %q; (-want, +got)\n%s", tc.desc, base.DefaultQueue, diff)
}
}
}
@@ -780,7 +781,7 @@ func TestDeleteDeadTask(t *testing.T) {
id xid.ID
score int64
want error
wantDead []*TaskMessage
wantDead []*base.TaskMessage
}{
{
dead: []sortedSetEntry{
@@ -790,7 +791,7 @@ func TestDeleteDeadTask(t *testing.T) {
id: m1.ID,
score: t1.Unix(),
want: nil,
wantDead: []*TaskMessage{m2},
wantDead: []*base.TaskMessage{m2},
},
{
dead: []sortedSetEntry{
@@ -800,14 +801,14 @@ func TestDeleteDeadTask(t *testing.T) {
id: m1.ID,
score: t2.Unix(), // id and score mismatch
want: ErrTaskNotFound,
wantDead: []*TaskMessage{m1, m2},
wantDead: []*base.TaskMessage{m1, m2},
},
{
dead: []sortedSetEntry{},
id: m1.ID,
score: t1.Unix(),
want: ErrTaskNotFound,
wantDead: []*TaskMessage{},
wantDead: []*base.TaskMessage{},
},
}
@@ -821,10 +822,10 @@ func TestDeleteDeadTask(t *testing.T) {
continue
}
gotDeadRaw := r.client.ZRange(deadQ, 0, -1).Val()
gotDeadRaw := r.client.ZRange(base.DeadQueue, 0, -1).Val()
gotDead := mustUnmarshalSlice(t, gotDeadRaw)
if diff := cmp.Diff(tc.wantDead, gotDead, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", deadQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.DeadQueue, diff)
}
}
}
@@ -841,7 +842,7 @@ func TestDeleteRetryTask(t *testing.T) {
id xid.ID
score int64
want error
wantRetry []*TaskMessage
wantRetry []*base.TaskMessage
}{
{
retry: []sortedSetEntry{
@@ -851,7 +852,7 @@ func TestDeleteRetryTask(t *testing.T) {
id: m1.ID,
score: t1.Unix(),
want: nil,
wantRetry: []*TaskMessage{m2},
wantRetry: []*base.TaskMessage{m2},
},
{
retry: []sortedSetEntry{
@@ -860,7 +861,7 @@ func TestDeleteRetryTask(t *testing.T) {
id: m2.ID,
score: t2.Unix(),
want: ErrTaskNotFound,
wantRetry: []*TaskMessage{m1},
wantRetry: []*base.TaskMessage{m1},
},
}
@@ -874,10 +875,10 @@ func TestDeleteRetryTask(t *testing.T) {
continue
}
gotRetryRaw := r.client.ZRange(retryQ, 0, -1).Val()
gotRetryRaw := r.client.ZRange(base.RetryQueue, 0, -1).Val()
gotRetry := mustUnmarshalSlice(t, gotRetryRaw)
if diff := cmp.Diff(tc.wantRetry, gotRetry, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", retryQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryQueue, diff)
}
}
}
@@ -894,7 +895,7 @@ func TestDeleteScheduledTask(t *testing.T) {
id xid.ID
score int64
want error
wantScheduled []*TaskMessage
wantScheduled []*base.TaskMessage
}{
{
scheduled: []sortedSetEntry{
@@ -904,7 +905,7 @@ func TestDeleteScheduledTask(t *testing.T) {
id: m1.ID,
score: t1.Unix(),
want: nil,
wantScheduled: []*TaskMessage{m2},
wantScheduled: []*base.TaskMessage{m2},
},
{
scheduled: []sortedSetEntry{
@@ -913,7 +914,7 @@ func TestDeleteScheduledTask(t *testing.T) {
id: m2.ID,
score: t2.Unix(),
want: ErrTaskNotFound,
wantScheduled: []*TaskMessage{m1},
wantScheduled: []*base.TaskMessage{m1},
},
}
@@ -927,10 +928,10 @@ func TestDeleteScheduledTask(t *testing.T) {
continue
}
gotScheduledRaw := r.client.ZRange(scheduledQ, 0, -1).Val()
gotScheduledRaw := r.client.ZRange(base.ScheduledQueue, 0, -1).Val()
gotScheduled := mustUnmarshalSlice(t, gotScheduledRaw)
if diff := cmp.Diff(tc.wantScheduled, gotScheduled, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", scheduledQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ScheduledQueue, diff)
}
}
}
@@ -943,7 +944,7 @@ func TestDeleteAllDeadTasks(t *testing.T) {
tests := []struct {
dead []sortedSetEntry
wantDead []*TaskMessage
wantDead []*base.TaskMessage
}{
{
dead: []sortedSetEntry{
@@ -951,7 +952,7 @@ func TestDeleteAllDeadTasks(t *testing.T) {
{m2, time.Now().Unix()},
{m3, time.Now().Unix()},
},
wantDead: []*TaskMessage{},
wantDead: []*base.TaskMessage{},
},
}
@@ -964,10 +965,10 @@ func TestDeleteAllDeadTasks(t *testing.T) {
t.Errorf("r.DeleteAllDeaadTasks = %v, want nil", err)
}
gotDeadRaw := r.client.ZRange(deadQ, 0, -1).Val()
gotDeadRaw := r.client.ZRange(base.DeadQueue, 0, -1).Val()
gotDead := mustUnmarshalSlice(t, gotDeadRaw)
if diff := cmp.Diff(tc.wantDead, gotDead, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", deadQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.DeadQueue, diff)
}
}
}
@@ -980,7 +981,7 @@ func TestDeleteAllRetryTasks(t *testing.T) {
tests := []struct {
retry []sortedSetEntry
wantRetry []*TaskMessage
wantRetry []*base.TaskMessage
}{
{
retry: []sortedSetEntry{
@@ -988,7 +989,7 @@ func TestDeleteAllRetryTasks(t *testing.T) {
{m2, time.Now().Unix()},
{m3, time.Now().Unix()},
},
wantRetry: []*TaskMessage{},
wantRetry: []*base.TaskMessage{},
},
}
@@ -1001,10 +1002,10 @@ func TestDeleteAllRetryTasks(t *testing.T) {
t.Errorf("r.DeleteAllDeaadTasks = %v, want nil", err)
}
gotRetryRaw := r.client.ZRange(retryQ, 0, -1).Val()
gotRetryRaw := r.client.ZRange(base.RetryQueue, 0, -1).Val()
gotRetry := mustUnmarshalSlice(t, gotRetryRaw)
if diff := cmp.Diff(tc.wantRetry, gotRetry, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", retryQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryQueue, diff)
}
}
}
@@ -1017,7 +1018,7 @@ func TestDeleteAllScheduledTasks(t *testing.T) {
tests := []struct {
scheduled []sortedSetEntry
wantScheduled []*TaskMessage
wantScheduled []*base.TaskMessage
}{
{
scheduled: []sortedSetEntry{
@@ -1025,7 +1026,7 @@ func TestDeleteAllScheduledTasks(t *testing.T) {
{m2, time.Now().Add(time.Minute).Unix()},
{m3, time.Now().Add(time.Minute).Unix()},
},
wantScheduled: []*TaskMessage{},
wantScheduled: []*base.TaskMessage{},
},
}
@@ -1038,10 +1039,10 @@ func TestDeleteAllScheduledTasks(t *testing.T) {
t.Errorf("r.DeleteAllDeaadTasks = %v, want nil", err)
}
gotScheduledRaw := r.client.ZRange(scheduledQ, 0, -1).Val()
gotScheduledRaw := r.client.ZRange(base.ScheduledQueue, 0, -1).Val()
gotScheduled := mustUnmarshalSlice(t, gotScheduledRaw)
if diff := cmp.Diff(tc.wantScheduled, gotScheduled, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", scheduledQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ScheduledQueue, diff)
}
}
}

86
internal/rdb/rdb.go
View File

@@ -8,17 +8,7 @@ import (
"time"
"github.com/go-redis/redis/v7"
"github.com/rs/xid"
)
// Redis keys
const (
queuePrefix = "asynq:queues:" // LIST - asynq:queues:<qname>
defaultQ = queuePrefix + "default" // LIST
scheduledQ = "asynq:scheduled" // ZSET
retryQ = "asynq:retry" // ZSET
deadQ = "asynq:dead" // ZSET
inProgressQ = "asynq:in_progress" // LIST
"github.com/hibiken/asynq/internal/base"
)
var (
@@ -39,28 +29,6 @@ func NewRDB(client *redis.Client) *RDB {
return &RDB{client}
}
// TaskMessage is the internal representation of a task with additional metadata fields.
// Serialized data of this type gets written in redis.
type TaskMessage struct {
//-------- Task fields --------
// Type represents the kind of task.
Type string
// Payload holds data needed to process the task.
Payload map[string]interface{}
//-------- Metadata fields --------
// ID is a unique identifier for each task
ID xid.ID
// Queue is a name this message should be enqueued to
Queue string
// Retry is the max number of retry for this task.
Retry int
// Retried is the number of times we've retried this task so far
Retried int
// ErrorMsg holds the error message from the last failure
ErrorMsg string
}
// Close closes the connection with redis server.
func (r *RDB) Close() error {
return r.client.Close()
@@ -68,12 +36,12 @@ func (r *RDB) Close() error {
// Enqueue inserts the given task to the end of the queue.
// It also adds the queue name to the "all-queues" list.
func (r *RDB) Enqueue(msg *TaskMessage) error {
func (r *RDB) Enqueue(msg *base.TaskMessage) error {
bytes, err := json.Marshal(msg)
if err != nil {
return fmt.Errorf("could not marshal %+v to json: %v", msg, err)
}
qname := queuePrefix + msg.Queue
qname := base.QueuePrefix + msg.Queue
pipe := r.client.Pipeline()
pipe.LPush(qname, string(bytes))
_, err = pipe.Exec()
@@ -86,15 +54,15 @@ func (r *RDB) Enqueue(msg *TaskMessage) error {
// Dequeue blocks until there is a task available to be processed,
// once a task is available, it adds the task to "in progress" list
// and returns the task.
func (r *RDB) Dequeue(timeout time.Duration) (*TaskMessage, error) {
data, err := r.client.BRPopLPush(defaultQ, inProgressQ, timeout).Result()
func (r *RDB) Dequeue(timeout time.Duration) (*base.TaskMessage, error) {
data, err := r.client.BRPopLPush(base.DefaultQueue, base.InProgressQueue, timeout).Result()
if err == redis.Nil {
return nil, ErrDequeueTimeout
}
if err != nil {
return nil, fmt.Errorf("command `BRPOPLPUSH %q %q %v` failed: %v", defaultQ, inProgressQ, timeout, err)
return nil, fmt.Errorf("command `BRPOPLPUSH %q %q %v` failed: %v", base.DefaultQueue, base.InProgressQueue, timeout, err)
}
var msg TaskMessage
var msg base.TaskMessage
err = json.Unmarshal([]byte(data), &msg)
if err != nil {
return nil, fmt.Errorf("could not unmarshal %v to json: %v", data, err)
@@ -103,22 +71,22 @@ func (r *RDB) Dequeue(timeout time.Duration) (*TaskMessage, error) {
}
// Done removes the task from in-progress queue to mark the task as done.
func (r *RDB) Done(msg *TaskMessage) error {
func (r *RDB) Done(msg *base.TaskMessage) error {
bytes, err := json.Marshal(msg)
if err != nil {
return fmt.Errorf("could not marshal %+v to json: %v", msg, err)
}
// NOTE: count ZERO means "remove all elements equal to val"
err = r.client.LRem(inProgressQ, 0, string(bytes)).Err()
err = r.client.LRem(base.InProgressQueue, 0, string(bytes)).Err()
if err != nil {
return fmt.Errorf("command `LREM %s 0 %s` failed: %v", inProgressQ, string(bytes), err)
return fmt.Errorf("command `LREM %s 0 %s` failed: %v", base.InProgressQueue, string(bytes), err)
}
return nil
}
// Requeue moves the task from in-progress queue to the default
// queue.
func (r *RDB) Requeue(msg *TaskMessage) error {
func (r *RDB) Requeue(msg *base.TaskMessage) error {
bytes, err := json.Marshal(msg)
if err != nil {
return fmt.Errorf("could not marshal %+v to json: %v", msg, err)
@@ -126,33 +94,33 @@ func (r *RDB) Requeue(msg *TaskMessage) error {
// Note: Use RPUSH to push to the head of the queue.
// KEYS[1] -> asynq:in_progress
// KEYS[2] -> asynq:queues:default
// ARGV[1] -> taskMessage value
// ARGV[1] -> base.TaskMessage value
script := redis.NewScript(`
redis.call("LREM", KEYS[1], 0, ARGV[1])
redis.call("RPUSH", KEYS[2], ARGV[1])
return redis.status_reply("OK")
`)
_, err = script.Run(r.client, []string{inProgressQ, defaultQ}, string(bytes)).Result()
_, err = script.Run(r.client, []string{base.InProgressQueue, base.DefaultQueue}, string(bytes)).Result()
return err
}
// Schedule adds the task to the backlog queue to be processed in the future.
func (r *RDB) Schedule(msg *TaskMessage, processAt time.Time) error {
func (r *RDB) Schedule(msg *base.TaskMessage, processAt time.Time) error {
bytes, err := json.Marshal(msg)
if err != nil {
return fmt.Errorf("could not marshal %+v to json: %v", msg, err)
}
score := float64(processAt.Unix())
err = r.client.ZAdd(scheduledQ, &redis.Z{Member: string(bytes), Score: score}).Err()
err = r.client.ZAdd(base.ScheduledQueue, &redis.Z{Member: string(bytes), Score: score}).Err()
if err != nil {
return fmt.Errorf("command `ZADD %s %.1f %s` failed: %v", scheduledQ, score, string(bytes), err)
return fmt.Errorf("command `ZADD %s %.1f %s` failed: %v", base.ScheduledQueue, score, string(bytes), err)
}
return nil
}
// Retry moves the task from in-progress to retry queue, incrementing retry count
// and assigning error message to the task message.
func (r *RDB) Retry(msg *TaskMessage, processAt time.Time, errMsg string) error {
func (r *RDB) Retry(msg *base.TaskMessage, processAt time.Time, errMsg string) error {
bytesToRemove, err := json.Marshal(msg)
if err != nil {
return fmt.Errorf("could not marshal %+v to json: %v", msg, err)
@@ -166,15 +134,15 @@ func (r *RDB) Retry(msg *TaskMessage, processAt time.Time, errMsg string) error
}
// KEYS[1] -> asynq:in_progress
// KEYS[2] -> asynq:retry
// ARGV[1] -> TaskMessage value to remove from InProgress queue
// ARGV[2] -> TaskMessage value to add to Retry queue
// ARGV[1] -> base.TaskMessage value to remove from base.InProgressQueue queue
// ARGV[2] -> base.TaskMessage value to add to Retry queue
// ARGV[3] -> retry_at UNIX timestamp
script := redis.NewScript(`
redis.call("LREM", KEYS[1], 0, ARGV[1])
redis.call("ZADD", KEYS[2], ARGV[3], ARGV[2])
return redis.status_reply("OK")
`)
_, err = script.Run(r.client, []string{inProgressQ, retryQ},
_, err = script.Run(r.client, []string{base.InProgressQueue, base.RetryQueue},
string(bytesToRemove), string(bytesToAdd), processAt.Unix()).Result()
return err
}
@@ -182,7 +150,7 @@ func (r *RDB) Retry(msg *TaskMessage, processAt time.Time, errMsg string) error
// Kill sends the task to "dead" queue from in-progress queue, assigning
// the error message to the task.
// It also trims the set by timestamp and set size.
func (r *RDB) Kill(msg *TaskMessage, errMsg string) error {
func (r *RDB) Kill(msg *base.TaskMessage, errMsg string) error {
const maxDeadTask = 10
const deadExpirationInDays = 90
bytesToRemove, err := json.Marshal(msg)
@@ -199,8 +167,8 @@ func (r *RDB) Kill(msg *TaskMessage, errMsg string) error {
limit := now.AddDate(0, 0, -deadExpirationInDays).Unix() // 90 days ago
// KEYS[1] -> asynq:in_progress
// KEYS[2] -> asynq:dead
// ARGV[1] -> TaskMessage value to remove from InProgress queue
// ARGV[2] -> TaskMessage value to add to Dead queue
// ARGV[1] -> base.TaskMessage value to remove from base.InProgressQueue queue
// ARGV[2] -> base.TaskMessage value to add to Dead queue
// ARGV[3] -> died_at UNIX timestamp
// ARGV[4] -> cutoff timestamp (e.g., 90 days ago)
// ARGV[5] -> max number of tasks in dead queue (e.g., 100)
@@ -211,7 +179,7 @@ func (r *RDB) Kill(msg *TaskMessage, errMsg string) error {
redis.call("ZREMRANGEBYRANK", KEYS[2], 0, -ARGV[5])
return redis.status_reply("OK")
`)
_, err = script.Run(r.client, []string{inProgressQ, deadQ},
_, err = script.Run(r.client, []string{base.InProgressQueue, base.DeadQueue},
string(bytesToRemove), string(bytesToAdd), now.Unix(), limit, maxDeadTask).Result()
return err
}
@@ -226,7 +194,7 @@ func (r *RDB) RestoreUnfinished() (int64, error) {
end
return len
`)
res, err := script.Run(r.client, []string{inProgressQ, defaultQ}).Result()
res, err := script.Run(r.client, []string{base.InProgressQueue, base.DefaultQueue}).Result()
if err != nil {
return 0, err
}
@@ -240,7 +208,7 @@ func (r *RDB) RestoreUnfinished() (int64, error) {
// CheckAndEnqueue checks for all scheduled tasks and enqueues any tasks that
// have to be processed.
func (r *RDB) CheckAndEnqueue() error {
delayed := []string{scheduledQ, retryQ}
delayed := []string{base.ScheduledQueue, base.RetryQueue}
for _, zset := range delayed {
if err := r.forward(zset); err != nil {
return err
@@ -261,6 +229,6 @@ func (r *RDB) forward(from string) error {
return msgs
`)
now := float64(time.Now().Unix())
_, err := script.Run(r.client, []string{from, defaultQ}, now).Result()
_, err := script.Run(r.client, []string{from, base.DefaultQueue}, now).Result()
return err
}

201
internal/rdb/rdb_test.go
View File

@@ -6,12 +6,13 @@ import (
"time"
"github.com/google/go-cmp/cmp"
"github.com/hibiken/asynq/internal/base"
)
func TestEnqueue(t *testing.T) {
r := setup(t)
tests := []struct {
msg *TaskMessage
msg *base.TaskMessage
}{
{msg: newTaskMessage("send_email", map[string]interface{}{"to": "exampleuser@gmail.com", "from": "noreply@example.com"})},
{msg: newTaskMessage("generate_csv", map[string]interface{}{})},
@@ -26,9 +27,9 @@ func TestEnqueue(t *testing.T) {
t.Errorf("(*RDB).Enqueue = %v, want nil", err)
continue
}
res := r.client.LRange(defaultQ, 0, -1).Val()
res := r.client.LRange(base.DefaultQueue, 0, -1).Val()
if len(res) != 1 {
t.Errorf("%q has length %d, want 1", defaultQ, len(res))
t.Errorf("%q has length %d, want 1", base.DefaultQueue, len(res))
continue
}
if diff := cmp.Diff(tc.msg, mustUnmarshal(t, res[0])); diff != "" {
@@ -41,13 +42,13 @@ func TestDequeue(t *testing.T) {
r := setup(t)
t1 := newTaskMessage("send_email", map[string]interface{}{"subject": "hello!"})
tests := []struct {
enqueued []*TaskMessage
want *TaskMessage
enqueued []*base.TaskMessage
want *base.TaskMessage
err error
inProgress int64 // length of "in-progress" tasks after dequeue
}{
{enqueued: []*TaskMessage{t1}, want: t1, err: nil, inProgress: 1},
{enqueued: []*TaskMessage{}, want: nil, err: ErrDequeueTimeout, inProgress: 0},
{enqueued: []*base.TaskMessage{t1}, want: t1, err: nil, inProgress: 1},
{enqueued: []*base.TaskMessage{}, want: nil, err: ErrDequeueTimeout, inProgress: 0},
}
for _, tc := range tests {
@@ -60,8 +61,8 @@ func TestDequeue(t *testing.T) {
got, err, tc.want, tc.err)
continue
}
if l := r.client.LLen(inProgressQ).Val(); l != tc.inProgress {
t.Errorf("%q has length %d, want %d", inProgressQ, l, tc.inProgress)
if l := r.client.LLen(base.InProgressQueue).Val(); l != tc.inProgress {
t.Errorf("%q has length %d, want %d", base.InProgressQueue, l, tc.inProgress)
}
}
}
@@ -72,24 +73,24 @@ func TestDone(t *testing.T) {
t2 := newTaskMessage("export_csv", nil)
tests := []struct {
inProgress []*TaskMessage // initial state of the in-progress list
target *TaskMessage // task to remove
wantInProgress []*TaskMessage // final state of the in-progress list
inProgress []*base.TaskMessage // initial state of the in-progress list
target *base.TaskMessage // task to remove
wantInProgress []*base.TaskMessage // final state of the in-progress list
}{
{
inProgress: []*TaskMessage{t1, t2},
inProgress: []*base.TaskMessage{t1, t2},
target: t1,
wantInProgress: []*TaskMessage{t2},
wantInProgress: []*base.TaskMessage{t2},
},
{
inProgress: []*TaskMessage{t2},
inProgress: []*base.TaskMessage{t2},
target: t1,
wantInProgress: []*TaskMessage{t2},
wantInProgress: []*base.TaskMessage{t2},
},
{
inProgress: []*TaskMessage{t1},
inProgress: []*base.TaskMessage{t1},
target: t1,
wantInProgress: []*TaskMessage{},
wantInProgress: []*base.TaskMessage{},
},
}
@@ -103,10 +104,10 @@ func TestDone(t *testing.T) {
continue
}
data := r.client.LRange(inProgressQ, 0, -1).Val()
data := r.client.LRange(base.InProgressQueue, 0, -1).Val()
gotInProgress := mustUnmarshalSlice(t, data)
if diff := cmp.Diff(tc.wantInProgress, gotInProgress, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q after calling (*RDB).Done: (-want, +got):\n%s", inProgressQ, diff)
t.Errorf("mismatch found in %q after calling (*RDB).Done: (-want, +got):\n%s", base.InProgressQueue, diff)
continue
}
}
@@ -118,25 +119,25 @@ func TestRequeue(t *testing.T) {
t2 := newTaskMessage("export_csv", nil)
tests := []struct {
enqueued []*TaskMessage // initial state of the default queue
inProgress []*TaskMessage // initial state of the in-progress list
target *TaskMessage // task to requeue
wantEnqueued []*TaskMessage // final state of the default queue
wantInProgress []*TaskMessage // final state of the in-progress list
enqueued []*base.TaskMessage // initial state of the default queue
inProgress []*base.TaskMessage // initial state of the in-progress list
target *base.TaskMessage // task to requeue
wantEnqueued []*base.TaskMessage // final state of the default queue
wantInProgress []*base.TaskMessage // final state of the in-progress list
}{
{
enqueued: []*TaskMessage{},
inProgress: []*TaskMessage{t1, t2},
enqueued: []*base.TaskMessage{},
inProgress: []*base.TaskMessage{t1, t2},
target: t1,
wantEnqueued: []*TaskMessage{t1},
wantInProgress: []*TaskMessage{t2},
wantEnqueued: []*base.TaskMessage{t1},
wantInProgress: []*base.TaskMessage{t2},
},
{
enqueued: []*TaskMessage{t1},
inProgress: []*TaskMessage{t2},
enqueued: []*base.TaskMessage{t1},
inProgress: []*base.TaskMessage{t2},
target: t2,
wantEnqueued: []*TaskMessage{t1, t2},
wantInProgress: []*TaskMessage{},
wantEnqueued: []*base.TaskMessage{t1, t2},
wantInProgress: []*base.TaskMessage{},
},
}
@@ -151,16 +152,16 @@ func TestRequeue(t *testing.T) {
continue
}
gotEnqueuedRaw := r.client.LRange(defaultQ, 0, -1).Val()
gotEnqueuedRaw := r.client.LRange(base.DefaultQueue, 0, -1).Val()
gotEnqueued := mustUnmarshalSlice(t, gotEnqueuedRaw)
if diff := cmp.Diff(tc.wantEnqueued, gotEnqueued, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q: (-want, +got):\n%s", defaultQ, diff)
t.Errorf("mismatch found in %q: (-want, +got):\n%s", base.DefaultQueue, diff)
}
gotInProgressRaw := r.client.LRange(inProgressQ, 0, -1).Val()
gotInProgressRaw := r.client.LRange(base.InProgressQueue, 0, -1).Val()
gotInProgress := mustUnmarshalSlice(t, gotInProgressRaw)
if diff := cmp.Diff(tc.wantInProgress, gotInProgress, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q: (-want, +got):\n%s", inProgressQ, diff)
t.Errorf("mismatch found in %q: (-want, +got):\n%s", base.InProgressQueue, diff)
}
}
}
@@ -171,7 +172,7 @@ func TestKill(t *testing.T) {
t2 := newTaskMessage("reindex", nil)
t3 := newTaskMessage("generate_csv", nil)
errMsg := "SMTP server not responding"
t1AfterKill := &TaskMessage{
t1AfterKill := &base.TaskMessage{
ID: t1.ID,
Type: t1.Type,
Payload: t1.Payload,
@@ -184,29 +185,29 @@ func TestKill(t *testing.T) {
// TODO(hibiken): add test cases for trimming
tests := []struct {
inProgress []*TaskMessage
inProgress []*base.TaskMessage
dead []sortedSetEntry
target *TaskMessage // task to kill
wantInProgress []*TaskMessage
target *base.TaskMessage // task to kill
wantInProgress []*base.TaskMessage
wantDead []sortedSetEntry
}{
{
inProgress: []*TaskMessage{t1, t2},
inProgress: []*base.TaskMessage{t1, t2},
dead: []sortedSetEntry{
{t3, now.Add(-time.Hour).Unix()},
},
target: t1,
wantInProgress: []*TaskMessage{t2},
wantInProgress: []*base.TaskMessage{t2},
wantDead: []sortedSetEntry{
{t1AfterKill, now.Unix()},
{t3, now.Add(-time.Hour).Unix()},
},
},
{
inProgress: []*TaskMessage{t1, t2, t3},
inProgress: []*base.TaskMessage{t1, t2, t3},
dead: []sortedSetEntry{},
target: t1,
wantInProgress: []*TaskMessage{t2, t3},
wantInProgress: []*base.TaskMessage{t2, t3},
wantDead: []sortedSetEntry{
{t1AfterKill, now.Unix()},
},
@@ -224,14 +225,14 @@ func TestKill(t *testing.T) {
continue
}
gotInProgressRaw := r.client.LRange(inProgressQ, 0, -1).Val()
gotInProgressRaw := r.client.LRange(base.InProgressQueue, 0, -1).Val()
gotInProgress := mustUnmarshalSlice(t, gotInProgressRaw)
if diff := cmp.Diff(tc.wantInProgress, gotInProgress, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", inProgressQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.InProgressQueue, diff)
}
var gotDead []sortedSetEntry
data := r.client.ZRangeWithScores(deadQ, 0, -1).Val()
data := r.client.ZRangeWithScores(base.DeadQueue, 0, -1).Val()
for _, z := range data {
gotDead = append(gotDead, sortedSetEntry{
msg: mustUnmarshal(t, z.Member.(string)),
@@ -241,7 +242,7 @@ func TestKill(t *testing.T) {
cmpOpt := cmp.AllowUnexported(sortedSetEntry{})
if diff := cmp.Diff(tc.wantDead, gotDead, cmpOpt, sortZSetEntryOpt); diff != "" {
t.Errorf("mismatch found in %q after calling (*RDB).Kill: (-want, +got):\n%s", deadQ, diff)
t.Errorf("mismatch found in %q after calling (*RDB).Kill: (-want, +got):\n%s", base.DeadQueue, diff)
}
}
}
@@ -253,32 +254,32 @@ func TestRestoreUnfinished(t *testing.T) {
t3 := newTaskMessage("sync_stuff", nil)
tests := []struct {
inProgress []*TaskMessage
enqueued []*TaskMessage
inProgress []*base.TaskMessage
enqueued []*base.TaskMessage
want int64
wantInProgress []*TaskMessage
wantEnqueued []*TaskMessage
wantInProgress []*base.TaskMessage
wantEnqueued []*base.TaskMessage
}{
{
inProgress: []*TaskMessage{t1, t2, t3},
enqueued: []*TaskMessage{},
inProgress: []*base.TaskMessage{t1, t2, t3},
enqueued: []*base.TaskMessage{},
want: 3,
wantInProgress: []*TaskMessage{},
wantEnqueued: []*TaskMessage{t1, t2, t3},
wantInProgress: []*base.TaskMessage{},
wantEnqueued: []*base.TaskMessage{t1, t2, t3},
},
{
inProgress: []*TaskMessage{},
enqueued: []*TaskMessage{t1, t2, t3},
inProgress: []*base.TaskMessage{},
enqueued: []*base.TaskMessage{t1, t2, t3},
want: 0,
wantInProgress: []*TaskMessage{},
wantEnqueued: []*TaskMessage{t1, t2, t3},
wantInProgress: []*base.TaskMessage{},
wantEnqueued: []*base.TaskMessage{t1, t2, t3},
},
{
inProgress: []*TaskMessage{t2, t3},
enqueued: []*TaskMessage{t1},
inProgress: []*base.TaskMessage{t2, t3},
enqueued: []*base.TaskMessage{t1},
want: 2,
wantInProgress: []*TaskMessage{},
wantEnqueued: []*TaskMessage{t1, t2, t3},
wantInProgress: []*base.TaskMessage{},
wantEnqueued: []*base.TaskMessage{t1, t2, t3},
},
}
@@ -294,15 +295,15 @@ func TestRestoreUnfinished(t *testing.T) {
continue
}
gotInProgressRaw := r.client.LRange(inProgressQ, 0, -1).Val()
gotInProgressRaw := r.client.LRange(base.InProgressQueue, 0, -1).Val()
gotInProgress := mustUnmarshalSlice(t, gotInProgressRaw)
if diff := cmp.Diff(tc.wantInProgress, gotInProgress, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q (-want, +got)\n%s", inProgressQ, diff)
t.Errorf("mismatch found in %q (-want, +got)\n%s", base.InProgressQueue, diff)
}
gotEnqueuedRaw := r.client.LRange(defaultQ, 0, -1).Val()
gotEnqueuedRaw := r.client.LRange(base.DefaultQueue, 0, -1).Val()
gotEnqueued := mustUnmarshalSlice(t, gotEnqueuedRaw)
if diff := cmp.Diff(tc.wantEnqueued, gotEnqueued, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q (-want, +got)\n%s", defaultQ, diff)
t.Errorf("mismatch found in %q (-want, +got)\n%s", base.DefaultQueue, diff)
}
}
}
@@ -318,9 +319,9 @@ func TestCheckAndEnqueue(t *testing.T) {
tests := []struct {
scheduled []sortedSetEntry
retry []sortedSetEntry
wantQueued []*TaskMessage
wantScheduled []*TaskMessage
wantRetry []*TaskMessage
wantQueued []*base.TaskMessage
wantScheduled []*base.TaskMessage
wantRetry []*base.TaskMessage
}{
{
scheduled: []sortedSetEntry{
@@ -328,9 +329,9 @@ func TestCheckAndEnqueue(t *testing.T) {
{t2, secondAgo.Unix()}},
retry: []sortedSetEntry{
{t3, secondAgo.Unix()}},
wantQueued: []*TaskMessage{t1, t2, t3},
wantScheduled: []*TaskMessage{},
wantRetry: []*TaskMessage{},
wantQueued: []*base.TaskMessage{t1, t2, t3},
wantScheduled: []*base.TaskMessage{},
wantRetry: []*base.TaskMessage{},
},
{
scheduled: []sortedSetEntry{
@@ -338,9 +339,9 @@ func TestCheckAndEnqueue(t *testing.T) {
{t2, secondAgo.Unix()}},
retry: []sortedSetEntry{
{t3, secondAgo.Unix()}},
wantQueued: []*TaskMessage{t2, t3},
wantScheduled: []*TaskMessage{t1},
wantRetry: []*TaskMessage{},
wantQueued: []*base.TaskMessage{t2, t3},
wantScheduled: []*base.TaskMessage{t1},
wantRetry: []*base.TaskMessage{},
},
{
scheduled: []sortedSetEntry{
@@ -348,9 +349,9 @@ func TestCheckAndEnqueue(t *testing.T) {
{t2, hourFromNow.Unix()}},
retry: []sortedSetEntry{
{t3, hourFromNow.Unix()}},
wantQueued: []*TaskMessage{},
wantScheduled: []*TaskMessage{t1, t2},
wantRetry: []*TaskMessage{t3},
wantQueued: []*base.TaskMessage{},
wantScheduled: []*base.TaskMessage{t1, t2},
wantRetry: []*base.TaskMessage{t3},
},
}
@@ -364,20 +365,20 @@ func TestCheckAndEnqueue(t *testing.T) {
t.Errorf("(*RDB).CheckScheduled() = %v, want nil", err)
continue
}
queued := r.client.LRange(defaultQ, 0, -1).Val()
queued := r.client.LRange(base.DefaultQueue, 0, -1).Val()
gotQueued := mustUnmarshalSlice(t, queued)
if diff := cmp.Diff(tc.wantQueued, gotQueued, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", defaultQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.DefaultQueue, diff)
}
scheduled := r.client.ZRange(scheduledQ, 0, -1).Val()
scheduled := r.client.ZRange(base.ScheduledQueue, 0, -1).Val()
gotScheduled := mustUnmarshalSlice(t, scheduled)
if diff := cmp.Diff(tc.wantScheduled, gotScheduled, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", scheduledQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.ScheduledQueue, diff)
}
retry := r.client.ZRange(retryQ, 0, -1).Val()
retry := r.client.ZRange(base.RetryQueue, 0, -1).Val()
gotRetry := mustUnmarshalSlice(t, retry)
if diff := cmp.Diff(tc.wantRetry, gotRetry, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", retryQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryQueue, diff)
}
}
}
@@ -385,7 +386,7 @@ func TestCheckAndEnqueue(t *testing.T) {
func TestSchedule(t *testing.T) {
r := setup(t)
tests := []struct {
msg *TaskMessage
msg *base.TaskMessage
processAt time.Time
}{
{
@@ -404,9 +405,9 @@ func TestSchedule(t *testing.T) {
continue
}
res := r.client.ZRangeWithScores(scheduledQ, 0, -1).Val()
res := r.client.ZRangeWithScores(base.ScheduledQueue, 0, -1).Val()
if len(res) != 1 {
t.Errorf("%s inserted %d items to %q, want 1 items inserted", desc, len(res), scheduledQ)
t.Errorf("%s inserted %d items to %q, want 1 items inserted", desc, len(res), base.ScheduledQueue)
continue
}
if res[0].Score != float64(tc.processAt.Unix()) {
@@ -423,7 +424,7 @@ func TestRetry(t *testing.T) {
t3 := newTaskMessage("reindex", nil)
t1.Retried = 10
errMsg := "SMTP server is not responding"
t1AfterRetry := &TaskMessage{
t1AfterRetry := &base.TaskMessage{
ID: t1.ID,
Type: t1.Type,
Payload: t1.Payload,
@@ -435,23 +436,23 @@ func TestRetry(t *testing.T) {
now := time.Now()
tests := []struct {
inProgress []*TaskMessage
inProgress []*base.TaskMessage
retry []sortedSetEntry
msg *TaskMessage
msg *base.TaskMessage
processAt time.Time
errMsg string
wantInProgress []*TaskMessage
wantInProgress []*base.TaskMessage
wantRetry []sortedSetEntry
}{
{
inProgress: []*TaskMessage{t1, t2},
inProgress: []*base.TaskMessage{t1, t2},
retry: []sortedSetEntry{
{t3, now.Add(time.Minute).Unix()},
},
msg: t1,
processAt: now.Add(5 * time.Minute),
errMsg: errMsg,
wantInProgress: []*TaskMessage{t2},
wantInProgress: []*base.TaskMessage{t2},
wantRetry: []sortedSetEntry{
{t1AfterRetry, now.Add(5 * time.Minute).Unix()},
{t3, now.Add(time.Minute).Unix()},
@@ -470,13 +471,13 @@ func TestRetry(t *testing.T) {
continue
}
gotInProgressRaw := r.client.LRange(inProgressQ, 0, -1).Val()
gotInProgressRaw := r.client.LRange(base.InProgressQueue, 0, -1).Val()
gotInProgress := mustUnmarshalSlice(t, gotInProgressRaw)
if diff := cmp.Diff(tc.wantInProgress, gotInProgress, sortMsgOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", inProgressQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.InProgressQueue, diff)
}
gotRetryRaw := r.client.ZRangeWithScores(retryQ, 0, -1).Val()
gotRetryRaw := r.client.ZRangeWithScores(base.RetryQueue, 0, -1).Val()
var gotRetry []sortedSetEntry
for _, z := range gotRetryRaw {
gotRetry = append(gotRetry, sortedSetEntry{
@@ -486,7 +487,7 @@ func TestRetry(t *testing.T) {
}
cmpOpt := cmp.AllowUnexported(sortedSetEntry{})
if diff := cmp.Diff(tc.wantRetry, gotRetry, cmpOpt, sortZSetEntryOpt); diff != "" {
t.Errorf("mismatch found in %q; (-want, +got)\n%s", retryQ, diff)
t.Errorf("mismatch found in %q; (-want, +got)\n%s", base.RetryQueue, diff)
}
}
}