go-pkg/asynq
2
0
Fork 0
mirror of https://github.com/hibiken/asynq.git synced 2025-10-18 08:57:55 +08:00
Code Issues Projects Releases Wiki Activity

feature: configurable janitor interval and deletion batch size (#715)

Browse Source 
* feature: configurable janitor interval and deletion batch size

* warn user when they set a big number of janitor batch size

* Update CHANGELOG.md

---------

Co-authored-by: Agung Hariadi Tedja <agung.tedja@kumparan.com>
This commit is contained in:
Tedja
2024-05-06 13:11:52 +07:00
committed by GitHub
parent 174008843d
commit d04888e748
8 changed files with 62 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/base/base.go
View File

@@ -737,7 +737,7 @@ type Broker interface {
ReclaimStaleAggregationSets(qname string) error
// Task retention related method
DeleteExpiredCompletedTasks(qname string) error
DeleteExpiredCompletedTasks(qname string, batchSize int) error
// Lease related methods
ListLeaseExpired(cutoff time.Time, qnames ...string) ([]*TaskMessage, error)

4
internal/rdb/rdb.go
View File

@@ -1241,9 +1241,7 @@ return table.getn(ids)`)
// DeleteExpiredCompletedTasks checks for any expired tasks in the given queue's completed set,
// and delete all expired tasks.
func (r *RDB) DeleteExpiredCompletedTasks(qname string) error {
// Note: Do this operation in fix batches to prevent long running script.
const batchSize = 100
func (r *RDB) DeleteExpiredCompletedTasks(qname string, batchSize int) error {
for {
n, err := r.deleteExpiredCompletedTasks(qname, batchSize)
if err != nil {

4
internal/rdb/rdb_test.go
View File

@@ -2542,8 +2542,8 @@ func TestDeleteExpiredCompletedTasks(t *testing.T) {
h.FlushDB(t, r.client)
h.SeedAllCompletedQueues(t, r.client, tc.completed)
if err := r.DeleteExpiredCompletedTasks(tc.qname); err != nil {
t.Errorf("DeleteExpiredCompletedTasks(%q) failed: %v", tc.qname, err)
if err := r.DeleteExpiredCompletedTasks(tc.qname, 100); err != nil {
t.Errorf("DeleteExpiredCompletedTasks(%q, 100) failed: %v", tc.qname, err)
continue
}

6
internal/testbroker/testbroker.go
View File

@@ -11,8 +11,8 @@ import (
"sync"
"time"
"github.com/redis/go-redis/v9"
"github.com/hibiken/asynq/internal/base"
"github.com/redis/go-redis/v9"
)
var errRedisDown = errors.New("testutil: redis is down")
@@ -145,13 +145,13 @@ func (tb *TestBroker) ForwardIfReady(qnames ...string) error {
return tb.real.ForwardIfReady(qnames...)
}
func (tb *TestBroker) DeleteExpiredCompletedTasks(qname string) error {
func (tb *TestBroker) DeleteExpiredCompletedTasks(qname string, batchSize int) error {
tb.mu.Lock()
defer tb.mu.Unlock()
if tb.sleeping {
return errRedisDown
}
return tb.real.DeleteExpiredCompletedTasks(qname)
return tb.real.DeleteExpiredCompletedTasks(qname, batchSize)
}
func (tb *TestBroker) ListLeaseExpired(cutoff time.Time, qnames ...string) ([]*base.TaskMessage, error) {