go-pkg/asynq
2
0
Fork 0
mirror of https://github.com/hibiken/asynq.git synced 2025-08-28 16:31:46 +08:00
Code Issues Projects Releases Wiki Activity

Refactor redis keys and store messages in protobuf

Browse Source 
Changes:
- Task messages are stored under "asynq:{<qname>}:t:<task_id>" key in redis, value is a HASH type and message are stored under "msg" key in the hash. The hash also stores "deadline", "timeout".
- Redis LIST and ZSET stores task message IDs
- Task messages are serialized using protocol buffer
This commit is contained in:
Ken Hibino
2021-03-12 16:23:08 -08:00
parent 2516c4baba
commit 7af3981929
22 changed files with 2534 additions and 668 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
inspeq/inspector.go
View File

@@ -548,11 +548,12 @@ func (i *Inspector) DeleteAllArchivedTasks(qname string) (int, error) {
}
// DeleteTaskByKey deletes a task with the given key from the given queue.
// TODO: We don't need score any more. Update this to delete task by ID
func (i *Inspector) DeleteTaskByKey(qname, key string) error {
if err := base.ValidateQueueName(qname); err != nil {
return err
}
prefix, id, score, err := parseTaskKey(key)
prefix, id, _, err := parseTaskKey(key)
if err != nil {
return err
}
@@ -560,11 +561,11 @@ func (i *Inspector) DeleteTaskByKey(qname, key string) error {
case keyPrefixPending:
return i.rdb.DeletePendingTask(qname, id)
case keyPrefixScheduled:
return i.rdb.DeleteScheduledTask(qname, id, score)
return i.rdb.DeleteScheduledTask(qname, id)
case keyPrefixRetry:
return i.rdb.DeleteRetryTask(qname, id, score)
return i.rdb.DeleteRetryTask(qname, id)
case keyPrefixArchived:
return i.rdb.DeleteArchivedTask(qname, id, score)
return i.rdb.DeleteArchivedTask(qname, id)
default:
return fmt.Errorf("invalid key")
}
@@ -601,21 +602,22 @@ func (i *Inspector) RunAllArchivedTasks(qname string) (int, error) {
}
// RunTaskByKey transition a task to pending state given task key and queue name.
// TODO: Update this to run task by ID.
func (i *Inspector) RunTaskByKey(qname, key string) error {
if err := base.ValidateQueueName(qname); err != nil {
return err
}
prefix, id, score, err := parseTaskKey(key)
prefix, id, _, err := parseTaskKey(key)
if err != nil {
return err
}
switch prefix {
case keyPrefixScheduled:
return i.rdb.RunScheduledTask(qname, id, score)
return i.rdb.RunScheduledTask(qname, id)
case keyPrefixRetry:
return i.rdb.RunRetryTask(qname, id, score)
return i.rdb.RunRetryTask(qname, id)
case keyPrefixArchived:
return i.rdb.RunArchivedTask(qname, id, score)
return i.rdb.RunArchivedTask(qname, id)
case keyPrefixPending:
return fmt.Errorf("task is already pending for run")
default:
@@ -654,11 +656,12 @@ func (i *Inspector) ArchiveAllRetryTasks(qname string) (int, error) {
}
// ArchiveTaskByKey archives a task with the given key in the given queue.
// TODO: Update this to Archive task by ID.
func (i *Inspector) ArchiveTaskByKey(qname, key string) error {
if err := base.ValidateQueueName(qname); err != nil {
return err
}
prefix, id, score, err := parseTaskKey(key)
prefix, id, _, err := parseTaskKey(key)
if err != nil {
return err
}
@@ -666,9 +669,9 @@ func (i *Inspector) ArchiveTaskByKey(qname, key string) error {
case keyPrefixPending:
return i.rdb.ArchivePendingTask(qname, id)
case keyPrefixScheduled:
return i.rdb.ArchiveScheduledTask(qname, id, score)
return i.rdb.ArchiveScheduledTask(qname, id)
case keyPrefixRetry:
return i.rdb.ArchiveRetryTask(qname, id, score)
return i.rdb.ArchiveRetryTask(qname, id)
case keyPrefixArchived:
return fmt.Errorf("task is already archived")
default:

13
inspeq/inspector_test.go
View File

@@ -518,8 +518,8 @@ func TestInspectorListPendingTasks(t *testing.T) {
defer r.Close()
m1 := h.NewTaskMessage("task1", nil)
m2 := h.NewTaskMessage("task2", nil)
m3 := h.NewTaskMessage("task3", nil)
m4 := h.NewTaskMessage("task4", nil)
m3 := h.NewTaskMessageWithQueue("task3", nil, "critical")
m4 := h.NewTaskMessageWithQueue("task4", nil, "low")
inspector := New(getRedisConnOpt(t))
@@ -587,8 +587,8 @@ func TestInspectorListActiveTasks(t *testing.T) {
defer r.Close()
m1 := h.NewTaskMessage("task1", nil)
m2 := h.NewTaskMessage("task2", nil)
m3 := h.NewTaskMessage("task3", nil)
m4 := h.NewTaskMessage("task4", nil)
m3 := h.NewTaskMessageWithQueue("task3", nil, "custom")
m4 := h.NewTaskMessageWithQueue("task4", nil, "custom")
inspector := New(getRedisConnOpt(t))
@@ -1254,13 +1254,13 @@ func TestInspectorArchiveAllPendingTasks(t *testing.T) {
},
{
pending: map[string][]*base.TaskMessage{
"default": {m3, m4},
"default": {m3},
},
archived: map[string][]base.Z{
"default": {z1, z2},
},
qname: "default",
want: 2,
want: 1,
wantPending: map[string][]*base.TaskMessage{
"default": {},
},
@@ -1269,7 +1269,6 @@ func TestInspectorArchiveAllPendingTasks(t *testing.T) {
z1,
z2,
base.Z{Message: m3, Score: now.Unix()},
base.Z{Message: m4, Score: now.Unix()},
},
},
},