go-pkg/asynq
2
0
Fork 0
mirror of https://github.com/hibiken/asynq.git synced 2025-10-03 05:12:01 +08:00
Code Issues Projects Releases Wiki Activity

[performance] Skip the overhead of json decoding when scheduling to one

Browse Source 
queue
This commit is contained in:
Ken Hibino
2020-01-14 07:26:41 -08:00
parent 5a6f737589
commit cb2ebf18ac
5 changed files with 50 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
internal/rdb/rdb.go
View File

@@ -292,10 +292,18 @@ func (r *RDB) RestoreUnfinished() (int64, error) {
// CheckAndEnqueue checks for all scheduled tasks and enqueues any tasks that
// have to be processed.
func (r *RDB) CheckAndEnqueue() error {
//
// qnames specifies to which queues to send tasks.
func (r *RDB) CheckAndEnqueue(qnames ...string) error {
delayed := []string{base.ScheduledQueue, base.RetryQueue}
for _, zset := range delayed {
if err := r.forward(zset); err != nil {
var err error
if len(qnames) == 1 {
err = r.forwardSingle(zset, base.QueueKey(qnames[0]))
} else {
err = r.forward(zset)
}
if err != nil {
return err
}
}
@@ -303,8 +311,8 @@ func (r *RDB) CheckAndEnqueue() error {
}
// forward moves all tasks with a score less than the current unix time
// from the given zset to the default queue.
func (r *RDB) forward(from string) error {
// from the src zset.
func (r *RDB) forward(src string) error {
script := redis.NewScript(`
local msgs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1])
for _, msg in ipairs(msgs) do
@@ -317,5 +325,21 @@ func (r *RDB) forward(from string) error {
`)
now := float64(time.Now().Unix())
return script.Run(r.client,
[]string{from, base.DefaultQueue}, now, base.QueuePrefix).Err()
[]string{src}, now, base.QueuePrefix).Err()
}
// forwardSingle moves all tasks with a score less than the current unix time
// from the src zset to dst list.
func (r *RDB) forwardSingle(src, dst string) error {
script := redis.NewScript(`
local msgs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1])
for _, msg in ipairs(msgs) do
redis.call("ZREM", KEYS[1], msg)
redis.call("LPUSH", KEYS[2], msg)
end
return msgs
`)
now := float64(time.Now().Unix())
return script.Run(r.client,
[]string{src, dst}, now).Err()
}

7
internal/rdb/rdb_test.go
View File

@@ -587,6 +587,7 @@ func TestCheckAndEnqueue(t *testing.T) {
tests := []struct {
scheduled []h.ZSetEntry
retry []h.ZSetEntry
qnames []string
wantEnqueued map[string][]*base.TaskMessage
wantScheduled []*base.TaskMessage
wantRetry []*base.TaskMessage
@@ -598,6 +599,7 @@ func TestCheckAndEnqueue(t *testing.T) {
},
retry: []h.ZSetEntry{
{Msg: t3, Score: float64(secondAgo.Unix())}},
qnames: []string{"default"},
wantEnqueued: map[string][]*base.TaskMessage{
"default": {t1, t2, t3},
},
@@ -610,6 +612,7 @@ func TestCheckAndEnqueue(t *testing.T) {
{Msg: t2, Score: float64(secondAgo.Unix())}},
retry: []h.ZSetEntry{
{Msg: t3, Score: float64(secondAgo.Unix())}},
qnames: []string{"default"},
wantEnqueued: map[string][]*base.TaskMessage{
"default": {t2, t3},
},
@@ -622,6 +625,7 @@ func TestCheckAndEnqueue(t *testing.T) {
{Msg: t2, Score: float64(hourFromNow.Unix())}},
retry: []h.ZSetEntry{
{Msg: t3, Score: float64(hourFromNow.Unix())}},
qnames: []string{"default"},
wantEnqueued: map[string][]*base.TaskMessage{
"default": {},
},
@@ -635,6 +639,7 @@ func TestCheckAndEnqueue(t *testing.T) {
},
retry: []h.ZSetEntry{
{Msg: t5, Score: float64(secondAgo.Unix())}},
qnames: []string{"default", "critical", "low"},
wantEnqueued: map[string][]*base.TaskMessage{
"default": {t1},
"critical": {t4},
@@ -650,7 +655,7 @@ func TestCheckAndEnqueue(t *testing.T) {
h.SeedScheduledQueue(t, r.client, tc.scheduled)
h.SeedRetryQueue(t, r.client, tc.retry)
err := r.CheckAndEnqueue()
err := r.CheckAndEnqueue(tc.qnames...)
if err != nil {
t.Errorf("(*RDB).CheckScheduled() = %v, want nil", err)
continue