go-pkg/asynq
2
0
Fork 0
mirror of https://github.com/hibiken/asynq.git synced 2025-10-03 17:22:01 +08:00
Code Issues Projects Releases Wiki Activity

Use zset for aggregation set to preserve score

Browse Source
This commit is contained in:
Ken Hibino
2022-03-11 10:59:05 -08:00
parent 8b16ede8bc
commit 3551d3334c
3 changed files with 25 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
internal/rdb/rdb.go
View File

@@ -1015,9 +1015,9 @@ if size == 0 then
end
local maxSize = tonumber(ARGV[1])
if maxSize ~= 0 and size >= maxSize then
local msgs = redis.call("ZRANGE", KEYS[1], 0, maxSize-1)
for _, msg in ipairs(msgs) do
redis.call("SADD", KEYS[2], msg)
local res = redis.call("ZRANGE", KEYS[1], 0, maxSize-1, "WITHSCORES")
for i=1, table.getn(res)-1, 2 do
redis.call("ZADD", KEYS[2], tonumber(res[i+1]), res[i])
end
redis.call("ZREMRANGEBYRANK", KEYS[1], 0, maxSize-1)
redis.call("ZADD", KEYS[3], ARGV[5], ARGV[4])
@@ -1030,9 +1030,9 @@ if maxDelay ~= 0 then
local oldestEntryScore = tonumber(oldestEntry[2])
local maxDelayTime = currentTime - maxDelay
if oldestEntryScore <= maxDelayTime then
local msgs = redis.call("ZRANGE", KEYS[1], 0, maxSize-1)
for _, msg in ipairs(msgs) do
redis.call("SADD", KEYS[2], msg)
local res = redis.call("ZRANGE", KEYS[1], 0, maxSize-1, "WITHSCORES")
for i=1, table.getn(res)-1, 2 do
redis.call("ZADD", KEYS[2], tonumber(res[i+1]), res[i])
end
redis.call("ZREMRANGEBYRANK", KEYS[1], 0, maxSize-1)
redis.call("ZADD", KEYS[3], ARGV[5], ARGV[4])
@@ -1043,9 +1043,9 @@ local latestEntry = redis.call("ZREVRANGE", KEYS[1], 0, 0, "WITHSCORES")
local latestEntryScore = tonumber(latestEntry[2])
local gracePeriodStartTime = currentTime - tonumber(ARGV[3])
if latestEntryScore <= gracePeriodStartTime then
local msgs = redis.call("ZRANGE", KEYS[1], 0, maxSize-1)
for _, msg in ipairs(msgs) do
redis.call("SADD", KEYS[2], msg)
local res = redis.call("ZRANGE", KEYS[1], 0, maxSize-1, "WITHSCORES")
for i=1, table.getn(res)-1, 2 do
redis.call("ZADD", KEYS[2], tonumber(res[i+1]), res[i])
end
redis.call("ZREMRANGEBYRANK", KEYS[1], 0, maxSize-1)
redis.call("ZADD", KEYS[3], ARGV[5], ARGV[4])
@@ -1101,7 +1101,7 @@ func (r *RDB) AggregationCheck(qname, gname string, t time.Time, gracePeriod, ma
// ARGV[1] -> task key prefix
var readAggregationSetCmd = redis.NewScript(`
local msgs = {}
local ids = redis.call("SMEMBERS", KEYS[1])
local ids = redis.call("ZRANGE", KEYS[1], 0, -1)
for _, id in ipairs(ids) do
local key = ARGV[1] .. id
table.insert(msgs, redis.call("HGET", key, "msg"))
@@ -1142,7 +1142,7 @@ func (r *RDB) ReadAggregationSet(qname, gname, setID string) ([]*base.TaskMessag
// -------
// ARGV[1] -> task key prefix
var deleteAggregationSetCmd = redis.NewScript(`
local ids = redis.call("SMEMBERS", KEYS[1])
local ids = redis.call("ZRANGE", KEYS[1], 0, -1)
for _, id in ipairs(ids) do
redis.call("DEL", ARGV[1] .. id)
end

19
internal/rdb/rdb_test.go
View File

@@ -3359,6 +3359,7 @@ func TestDeleteAggregationSet(t *testing.T) {
r := setup(t)
defer r.Close()
now := time.Now()
ctx := context.Background()
setID := uuid.NewString()
msg1 := h.NewTaskMessageBuilder().SetType("foo").SetQueue("default").SetGroup("mygroup").Build()
@@ -3366,16 +3367,20 @@ func TestDeleteAggregationSet(t *testing.T) {
msg3 := h.NewTaskMessageBuilder().SetType("baz").SetQueue("default").SetGroup("mygroup").Build()
tests := []struct {
aggregationSet []*base.TaskMessage
aggregationSet []base.Z
qname string
gname string
setID string
}{
{
aggregationSet: []*base.TaskMessage{msg1, msg2, msg3},
qname: "default",
gname: "mygroup",
setID: setID,
aggregationSet: []base.Z{
{msg1, now.Add(-3 * time.Minute).Unix()},
{msg2, now.Add(-2 * time.Minute).Unix()},
{msg3, now.Add(-1 * time.Minute).Unix()},
},
qname: "default",
gname: "mygroup",
setID: setID,
},
}
@@ -3393,8 +3398,8 @@ func TestDeleteAggregationSet(t *testing.T) {
}
// Check all tasks in the set are deleted.
for _, m := range tc.aggregationSet {
taskKey := base.TaskKey(m.Queue, m.ID)
for _, z := range tc.aggregationSet {
taskKey := base.TaskKey(z.Message.Queue, z.Message.ID)
if r.client.Exists(ctx, taskKey).Val() != 0 {
t.Errorf("task key %q still exists", taskKey)
}