feature: configurable archive ttl and max archived size

internal/base/base.go

@@ -31,6 +31,11 @@ const DefaultQueueName = "default"
 // DefaultQueue is the redis key for the default queue.
 var DefaultQueue = PendingKey(DefaultQueueName)
 
+const (
+	DefaultMaxArchiveSize           = 10000 // maximum number of tasks in archive
+	DefaultArchivedExpirationInDays = 90    // number of days before an archived task gets deleted permanently
+)
+
 // Global Redis keys.
 const (
 	AllServers    = "asynq:servers"    // ZSET

internal/rdb/inspect.go

@@ -1181,8 +1181,8 @@ func (r *RDB) ArchiveAllAggregatingTasks(qname, gname string) (int64, error) {
 	now := r.clock.Now()
 	argv := []interface{}{
 		now.Unix(),
-		now.AddDate(0, 0, -archivedExpirationInDays).Unix(),
-		maxArchiveSize,
+		now.AddDate(0, 0, -(*r.config.ArchivedExpirationInDays)).Unix(),
+		*r.config.MaxArchiveSize,
 		base.TaskKeyPrefix(qname),
 		gname,
 	}
@@ -1237,8 +1237,8 @@ func (r *RDB) ArchiveAllPendingTasks(qname string) (int64, error) {
 	now := r.clock.Now()
 	argv := []interface{}{
 		now.Unix(),
-		now.AddDate(0, 0, -archivedExpirationInDays).Unix(),
-		maxArchiveSize,
+		now.AddDate(0, 0, -(*r.config.ArchivedExpirationInDays)).Unix(),
+		*r.config.MaxArchiveSize,
 		base.TaskKeyPrefix(qname),
 	}
 	res, err := archiveAllPendingCmd.Run(context.Background(), r.client, keys, argv...).Result()
@@ -1328,8 +1328,8 @@ func (r *RDB) ArchiveTask(qname, id string) error {
 	argv := []interface{}{
 		id,
 		now.Unix(),
-		now.AddDate(0, 0, -archivedExpirationInDays).Unix(),
-		maxArchiveSize,
+		now.AddDate(0, 0, -(*r.config.ArchivedExpirationInDays)).Unix(),
+		*r.config.MaxArchiveSize,
 		base.QueueKeyPrefix(qname),
 		base.GroupKeyPrefix(qname),
 	}
@@ -1393,8 +1393,8 @@ func (r *RDB) archiveAll(src, dst, qname string) (int64, error) {
 	now := r.clock.Now()
 	argv := []interface{}{
 		now.Unix(),
-		now.AddDate(0, 0, -archivedExpirationInDays).Unix(),
-		maxArchiveSize,
+		now.AddDate(0, 0, -(*r.config.ArchivedExpirationInDays)).Unix(),
+		*r.config.MaxArchiveSize,
 		base.TaskKeyPrefix(qname),
 		qname,
 	}

internal/rdb/rdb.go

@@ -30,14 +30,42 @@ type RDB struct {
 	client          redis.UniversalClient
 	clock           timeutil.Clock
 	queuesPublished sync.Map
+	config          RDBConfig
+}
+type RDBConfig struct {
+	MaxArchiveSize           *int
+	ArchivedExpirationInDays *int
+}
+
+func validateRDBConfig(cfg *RDBConfig) {
+	if cfg.MaxArchiveSize == nil {
+		value := base.DefaultMaxArchiveSize
+		cfg.MaxArchiveSize = &value
+	}
+	if cfg.ArchivedExpirationInDays == nil {
+		value := base.DefaultArchivedExpirationInDays
+		cfg.ArchivedExpirationInDays = &value
+	}
+	if *(cfg.ArchivedExpirationInDays) < 0 {
+		value := 1
+		cfg.ArchivedExpirationInDays = &value
+	}
+}
+
+// NewRDBWithConfig returns a new instance of RDB.
+func NewRDBWithConfig(client redis.UniversalClient, cfg RDBConfig) *RDB {
+	validateRDBConfig(&cfg)
+
+	return &RDB{
+		client: client,
+		clock:  timeutil.NewRealClock(),
+		config: cfg,
+	}
 }
 
 // NewRDB returns a new instance of RDB.
 func NewRDB(client redis.UniversalClient) *RDB {
-	return &RDB{
-		client: client,
-		clock:  timeutil.NewRealClock(),
-	}
+	return NewRDBWithConfig(client, RDBConfig{})
 }
 
 // Close closes the connection with redis server.
@@ -836,11 +864,6 @@ func (r *RDB) Retry(ctx context.Context, msg *base.TaskMessage, processAt time.T
 	return r.runScript(ctx, op, retryCmd, keys, argv...)
 }
 
-const (
-	maxArchiveSize           = 10000 // maximum number of tasks in archive
-	archivedExpirationInDays = 90    // number of days before an archived task gets deleted permanently
-)
-
 // KEYS[1] -> asynq:{<qname>}:t:<task_id>
 // KEYS[2] -> asynq:{<qname>}:active
 // KEYS[3] -> asynq:{<qname>}:lease
@@ -904,6 +927,10 @@ return redis.status_reply("OK")`)
 // Archive sends the given task to archive, attaching the error message to the task.
 // It also trims the archive by timestamp and set size.
 func (r *RDB) Archive(ctx context.Context, msg *base.TaskMessage, errMsg string) error {
+	if *(r.config.MaxArchiveSize) <= 0 {
+		return nil
+	}
+
 	var op errors.Op = "rdb.Archive"
 	now := r.clock.Now()
 	modified := *msg
@@ -913,7 +940,8 @@ func (r *RDB) Archive(ctx context.Context, msg *base.TaskMessage, errMsg string)
 	if err != nil {
 		return errors.E(op, errors.Internal, fmt.Sprintf("cannot encode message: %v", err))
 	}
-	cutoff := now.AddDate(0, 0, -archivedExpirationInDays)
+	cutoff := now.AddDate(0, 0, -(*r.config.ArchivedExpirationInDays))
+
 	expireAt := now.Add(statsTTL)
 	keys := []string{
 		base.TaskKey(msg.Queue, msg.ID),
@@ -931,7 +959,7 @@ func (r *RDB) Archive(ctx context.Context, msg *base.TaskMessage, errMsg string)
 		encoded,
 		now.Unix(),
 		cutoff.Unix(),
-		maxArchiveSize,
+		*r.config.MaxArchiveSize,
 		expireAt.Unix(),
 		int64(math.MaxInt64),
 	}

internal/rdb/rdb_test.go

@@ -2250,7 +2250,7 @@ func TestArchiveTrim(t *testing.T) {
 	errMsg := "SMTP server not responding"
 
 	maxArchiveSet := make([]base.Z, 0)
-	for i := 0; i < maxArchiveSize-1; i++ {
+	for i := 0; i < base.DefaultMaxArchiveSize-1; i++ {
 		maxArchiveSet = append(maxArchiveSet, base.Z{Message: &base.TaskMessage{
 			ID:      uuid.NewString(),
 			Type:    "generate_csv",
@@ -2306,7 +2306,7 @@ func TestArchiveTrim(t *testing.T) {
 			},
 			archived: map[string][]base.Z{
 				"default": {
-					{Message: t2, Score: now.Add(-time.Hour * 24 * (archivedExpirationInDays + 1)).Unix()},
+					{Message: t2, Score: now.Add(-time.Hour * 24 * (base.DefaultArchivedExpirationInDays + 1)).Unix()},
 				},
 			},
 			wantArchived: map[string][]base.Z{