v0.11.0

The command converts all messages in redis to be compatible for asynq
v0.10.0

.travis.yml

@@ -5,6 +5,7 @@ git:
 go: [1.13.x, 1.14.x]
 script:
   - go test -race -v -coverprofile=coverage.txt -covermode=atomic ./...
+  - go test -run=XXX -bench=. -loglevel=debug ./...
 services:
   - redis-server
 after_success:

.travis/benchcmp.sh

@@ -3,13 +3,16 @@ if [ "${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" != "master" ]; then
     cd ${TRAVIS_BUILD_DIR}/.. && \
     git clone ${REMOTE_URL} "${TRAVIS_REPO_SLUG}-bench" && \
     cd "${TRAVIS_REPO_SLUG}-bench" && \
+
     # Benchmark master
     git checkout master && \
     go test -run=XXX -bench=. ./... > master.txt && \
+
     # Benchmark feature branch
     git checkout ${TRAVIS_COMMIT} && \
     go test -run=XXX -bench=. ./... > feature.txt && \
-    go get -u golang.org/x/tools/cmd/benchcmp && \
+
     # compare two benchmarks
+    go get -u golang.org/x/tools/cmd/benchcmp && \
     benchcmp master.txt feature.txt;
-fi
+fi

CHANGELOG.md

@@ -7,6 +7,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.11.0] - 2020-07-28
+
+### Added
+
+- `Inspector` type was added to monitor and mutate state of queues and tasks.
+- `HealthCheckFunc` and `HealthCheckInterval` fields were added to `Config` to allow user to specify a callback
+  function to check for broker connection.
+
+## [0.10.0] - 2020-07-06
+
+### Changed
+
+- All tasks now requires timeout or deadline. By default, timeout is set to 30 mins.
+- Tasks that exceed its deadline are automatically retried.
+- Encoding schema for task message has changed. Please install the latest CLI and run `migrate` command if 
+  you have tasks enqueued with the previous version of asynq.
+- API of `(*Client).Enqueue`, `(*Client).EnqueueIn`, and `(*Client).EnqueueAt` has changed to return a `*Result`.
+- API of `ErrorHandler` has changed. It now takes context as the first argument and removed `retried`, `maxRetry` from the argument list.
+  Use `GetRetryCount` and/or `GetMaxRetry` to get the count values.
+
+## [0.9.4] - 2020-06-13
+
+### Fixed
+
+- Fixes issue of same tasks processed by more than one worker (https://github.com/hibiken/asynq/issues/90).
+
+## [0.9.3] - 2020-06-12
+
+### Fixed
+
+- Fixes the JSON number overflow issue (https://github.com/hibiken/asynq/issues/166).
+
+
+## [0.9.2] - 2020-06-08
+
+### Added
+
+- The `pause` and `unpause` commands were added to the CLI. See README for the CLI for details.
+
 ## [0.9.1] - 2020-05-29
 
 ### Added

README.md

@@ -34,12 +34,14 @@ A system can consist of multiple worker servers and brokers, giving way to high
 - Scheduling of tasks
 - Durability since tasks are written to Redis
 - [Retries](https://github.com/hibiken/asynq/wiki/Task-Retry) of failed tasks
+- Automatic recovery of tasks in the event of a worker crash
 - [Weighted priority queues](https://github.com/hibiken/asynq/wiki/Priority-Queues#weighted-priority-queues)
 - [Strict priority queues](https://github.com/hibiken/asynq/wiki/Priority-Queues#strict-priority-queues)
 - Low latency to add a task since writes are fast in Redis
 - De-duplication of tasks using [unique option](https://github.com/hibiken/asynq/wiki/Unique-Tasks)
 - Allow [timeout and deadline per task](https://github.com/hibiken/asynq/wiki/Task-Timeout-and-Cancelation)
 - [Flexible handler interface with support for middlewares](https://github.com/hibiken/asynq/wiki/Handler-Deep-Dive)
+- [Ability to pause queue](/tools/asynq/README.md#pause) to stop processing tasks from the queue
 - [Support Redis Sentinels](https://github.com/hibiken/asynq/wiki/Automatic-Failover) for HA
 - [CLI](#command-line-tool) to inspect and remote-control queues and tasks
 
@@ -156,10 +158,11 @@ func main() {
     // ------------------------------------------------------
 
     t := tasks.NewEmailDeliveryTask(42, "some:template:id")
-    err := c.Enqueue(t)
+    res, err := c.Enqueue(t)
     if err != nil {
         log.Fatal("could not enqueue task: %v", err)
     }
+    fmt.Printf("Enqueued Result: %+v\n", res)
 
 
     // ------------------------------------------------------------
@@ -168,10 +171,11 @@ func main() {
     // ------------------------------------------------------------
 
     t = tasks.NewEmailDeliveryTask(42, "other:template:id")
-    err = c.EnqueueIn(24*time.Hour, t)
+    res, err = c.EnqueueIn(24*time.Hour, t)
     if err != nil {
         log.Fatal("could not schedule task: %v", err)
     }
+    fmt.Printf("Enqueued Result: %+v\n", res)
 
 
     // ----------------------------------------------------------------------------
@@ -182,10 +186,11 @@ func main() {
     c.SetDefaultOptions(tasks.ImageProcessing, asynq.MaxRetry(10), asynq.Timeout(time.Minute))
 
     t = tasks.NewImageProcessingTask("some/blobstore/url", "other/blobstore/url")
-    err = c.Enqueue(t)
+    res, err = c.Enqueue(t)
     if err != nil {
         log.Fatal("could not enqueue task: %v", err)
     }
+    fmt.Printf("Enqueued Result: %+v\n", res)
 
     // ---------------------------------------------------------------------------
     // Example 4: Pass options to tune task processing behavior at enqueue time.
@@ -193,10 +198,11 @@ func main() {
     // ---------------------------------------------------------------------------
 
     t = tasks.NewImageProcessingTask("some/blobstore/url", "other/blobstore/url")
-    err = c.Enqueue(t, asynq.Queue("critical"), asynq.Timeout(30*time.Second))
+    res, err = c.Enqueue(t, asynq.Queue("critical"), asynq.Timeout(30*time.Second))
     if err != nil {
         log.Fatal("could not enqueue task: %v", err)
     }
+    fmt.Printf("Enqueued Result: %+v\n", res)
 }
 ```
 

benchmark_test.go

@@ -7,7 +7,6 @@ package asynq
 import (
 	"context"
 	"fmt"
-	"math/rand"
 	"sync"
 	"testing"
 	"time"
@@ -29,11 +28,12 @@ func BenchmarkEndToEndSimple(b *testing.B) {
 			RetryDelayFunc: func(n int, err error, t *Task) time.Duration {
 				return time.Second
 			},
+			LogLevel: testLogLevel,
 		})
 		// Create a bunch of tasks
 		for i := 0; i < count; i++ {
 			t := NewTask(fmt.Sprintf("task%d", i), map[string]interface{}{"data": i})
-			if err := client.Enqueue(t); err != nil {
+			if _, err := client.Enqueue(t); err != nil {
 				b.Fatalf("could not enqueue a task: %v", err)
 			}
 		}
@@ -60,7 +60,6 @@ func BenchmarkEndToEnd(b *testing.B) {
 	const count = 100000
 	for n := 0; n < b.N; n++ {
 		b.StopTimer() // begin setup
-		rand.Seed(time.Now().UnixNano())
 		setup(b)
 		redis := &RedisClientOpt{
 			Addr: redisAddr,
@@ -72,17 +71,18 @@ func BenchmarkEndToEnd(b *testing.B) {
 			RetryDelayFunc: func(n int, err error, t *Task) time.Duration {
 				return time.Second
 			},
+			LogLevel: testLogLevel,
 		})
 		// Create a bunch of tasks
 		for i := 0; i < count; i++ {
 			t := NewTask(fmt.Sprintf("task%d", i), map[string]interface{}{"data": i})
-			if err := client.Enqueue(t); err != nil {
+			if _, err := client.Enqueue(t); err != nil {
 				b.Fatalf("could not enqueue a task: %v", err)
 			}
 		}
 		for i := 0; i < count; i++ {
 			t := NewTask(fmt.Sprintf("scheduled%d", i), map[string]interface{}{"data": i})
-			if err := client.EnqueueAt(time.Now().Add(time.Second), t); err != nil {
+			if _, err := client.EnqueueAt(time.Now().Add(time.Second), t); err != nil {
 				b.Fatalf("could not enqueue a task: %v", err)
 			}
 		}
@@ -90,8 +90,16 @@ func BenchmarkEndToEnd(b *testing.B) {
 		var wg sync.WaitGroup
 		wg.Add(count * 2)
 		handler := func(ctx context.Context, t *Task) error {
-			// randomly fail 1% of tasks
-			if rand.Intn(100) == 1 {
+			n, err := t.Payload.GetInt("data")
+			if err != nil {
+				b.Logf("internal error: %v", err)
+			}
+			retried, ok := GetRetryCount(ctx)
+			if !ok {
+				b.Logf("internal error: %v", err)
+			}
+			// Fail 1% of tasks for the first attempt.
+			if retried == 0 && n%100 == 0 {
 				return fmt.Errorf(":(")
 			}
 			wg.Done()
@@ -131,23 +139,24 @@ func BenchmarkEndToEndMultipleQueues(b *testing.B) {
 				"default": 3,
 				"low":     1,
 			},
+			LogLevel: testLogLevel,
 		})
 		// Create a bunch of tasks
 		for i := 0; i < highCount; i++ {
 			t := NewTask(fmt.Sprintf("task%d", i), map[string]interface{}{"data": i})
-			if err := client.Enqueue(t, Queue("high")); err != nil {
+			if _, err := client.Enqueue(t, Queue("high")); err != nil {
 				b.Fatalf("could not enqueue a task: %v", err)
 			}
 		}
 		for i := 0; i < defaultCount; i++ {
 			t := NewTask(fmt.Sprintf("task%d", i), map[string]interface{}{"data": i})
-			if err := client.Enqueue(t); err != nil {
+			if _, err := client.Enqueue(t); err != nil {
 				b.Fatalf("could not enqueue a task: %v", err)
 			}
 		}
 		for i := 0; i < lowCount; i++ {
 			t := NewTask(fmt.Sprintf("task%d", i), map[string]interface{}{"data": i})
-			if err := client.Enqueue(t, Queue("low")); err != nil {
+			if _, err := client.Enqueue(t, Queue("low")); err != nil {
 				b.Fatalf("could not enqueue a task: %v", err)
 			}
 		}
@@ -168,3 +177,62 @@ func BenchmarkEndToEndMultipleQueues(b *testing.B) {
 		b.StartTimer() // end teardown
 	}
 }
+
+// E2E benchmark to check client enqueue operation performs correctly,
+// while server is busy processing tasks.
+func BenchmarkClientWhileServerRunning(b *testing.B) {
+	const count = 10000
+	for n := 0; n < b.N; n++ {
+		b.StopTimer() // begin setup
+		setup(b)
+		redis := &RedisClientOpt{
+			Addr: redisAddr,
+			DB:   redisDB,
+		}
+		client := NewClient(redis)
+		srv := NewServer(redis, Config{
+			Concurrency: 10,
+			RetryDelayFunc: func(n int, err error, t *Task) time.Duration {
+				return time.Second
+			},
+			LogLevel: testLogLevel,
+		})
+		// Enqueue 10,000 tasks.
+		for i := 0; i < count; i++ {
+			t := NewTask(fmt.Sprintf("task%d", i), map[string]interface{}{"data": i})
+			if _, err := client.Enqueue(t); err != nil {
+				b.Fatalf("could not enqueue a task: %v", err)
+			}
+		}
+		// Schedule 10,000 tasks.
+		for i := 0; i < count; i++ {
+			t := NewTask(fmt.Sprintf("scheduled%d", i), map[string]interface{}{"data": i})
+			if _, err := client.EnqueueAt(time.Now().Add(time.Second), t); err != nil {
+				b.Fatalf("could not enqueue a task: %v", err)
+			}
+		}
+
+		handler := func(ctx context.Context, t *Task) error {
+			return nil
+		}
+		srv.Start(HandlerFunc(handler))
+
+		b.StartTimer() // end setup
+
+		b.Log("Starting enqueueing")
+		enqueued := 0
+		for enqueued < 100000 {
+			t := NewTask(fmt.Sprintf("enqueued%d", enqueued), map[string]interface{}{"data": enqueued})
+			if _, err := client.Enqueue(t); err != nil {
+				b.Logf("could not enqueue task %d: %v", enqueued, err)
+				continue
+			}
+			enqueued++
+		}
+		b.Logf("Finished enqueueing %d tasks", enqueued)
+
+		b.StopTimer() // begin teardown
+		srv.Stop()
+		b.StartTimer() // end teardown
+	}
+}

client.go

@@ -12,9 +12,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/hibiken/asynq/internal/base"
 	"github.com/hibiken/asynq/internal/rdb"
-	"github.com/rs/xid"
 )
 
 // A Client is responsible for scheduling tasks.
@@ -69,13 +69,23 @@ func Queue(name string) Option {
 }
 
 // Timeout returns an option to specify how long a task may run.
+// If the timeout elapses before the Handler returns, then the task
+// will be retried.
 //
 // Zero duration means no limit.
+//
+// If there's a conflicting Deadline option, whichever comes earliest
+// will be used.
 func Timeout(d time.Duration) Option {
 	return timeoutOption(d)
 }
 
 // Deadline returns an option to specify the deadline for the given task.
+// If it reaches the deadline before the Handler returns, then the task
+// will be retried.
+//
+// If there's a conflicting Timeout option, whichever comes earliest
+// will be used.
 func Deadline(t time.Time) Option {
 	return deadlineOption(t)
 }
@@ -110,7 +120,7 @@ func composeOptions(opts ...Option) option {
 	res := option{
 		retry:    defaultMaxRetry,
 		queue:    base.DefaultQueueName,
-		timeout:  0,
+		timeout:  0, // do not set to deafultTimeout here
 		deadline: time.Time{},
 	}
 	for _, opt := range opts {
@@ -165,8 +175,19 @@ func serializePayload(payload map[string]interface{}) string {
 	return b.String()
 }
 
-// Default max retry count used if nothing is specified.
-const defaultMaxRetry = 25
+const (
+	// Default max retry count used if nothing is specified.
+	defaultMaxRetry = 25
+
+	// Default timeout used if both timeout and deadline are not specified.
+	defaultTimeout = 30 * time.Minute
+)
+
+// Value zero indicates no timeout and no deadline.
+var (
+	noTimeout  time.Duration = 0
+	noDeadline time.Time     = time.Unix(0, 0)
+)
 
 // SetDefaultOptions sets options to be used for a given task type.
 // The argument opts specifies the behavior of task processing.
@@ -179,13 +200,43 @@ func (c *Client) SetDefaultOptions(taskType string, opts ...Option) {
 	c.opts[taskType] = opts
 }
 
+// A Result holds enqueued task's metadata.
+type Result struct {
+	// ID is a unique identifier for the task.
+	ID string
+
+	// Retry is the maximum number of retry for the task.
+	Retry int
+
+	// Queue is a name of the queue the task is enqueued to.
+	Queue string
+
+	// Timeout is the timeout value for the task.
+	// Counting for timeout starts when a worker starts processing the task.
+	// If task processing doesn't complete within the timeout, the task will be retried.
+	// The value zero means no timeout.
+	//
+	// If deadline is set, min(now+timeout, deadline) is used, where the now is the time when
+	// a worker starts processing the task.
+	Timeout time.Duration
+
+	// Deadline is the deadline value for the task.
+	// If task processing doesn't complete before the deadline, the task will be retried.
+	// The value time.Unix(0, 0) means no deadline.
+	//
+	// If timeout is set, min(now+timeout, deadline) is used, where the now is the time when
+	// a worker starts processing the task.
+	Deadline time.Time
+}
+
 // EnqueueAt schedules task to be enqueued at the specified time.
 //
 // EnqueueAt returns nil if the task is scheduled successfully, otherwise returns a non-nil error.
 //
 // The argument opts specifies the behavior of task processing.
 // If there are conflicting Option values the last one overrides others.
-func (c *Client) EnqueueAt(t time.Time, task *Task, opts ...Option) error {
+// By deafult, max retry is set to 25 and timeout is set to 30 minutes.
+func (c *Client) EnqueueAt(t time.Time, task *Task, opts ...Option) (*Result, error) {
 	return c.enqueueAt(t, task, opts...)
 }
 
@@ -195,7 +246,8 @@ func (c *Client) EnqueueAt(t time.Time, task *Task, opts ...Option) error {
 //
 // The argument opts specifies the behavior of task processing.
 // If there are conflicting Option values the last one overrides others.
-func (c *Client) Enqueue(task *Task, opts ...Option) error {
+// By deafult, max retry is set to 25 and timeout is set to 30 minutes.
+func (c *Client) Enqueue(task *Task, opts ...Option) (*Result, error) {
 	return c.enqueueAt(time.Now(), task, opts...)
 }
 
@@ -205,7 +257,8 @@ func (c *Client) Enqueue(task *Task, opts ...Option) error {
 //
 // The argument opts specifies the behavior of task processing.
 // If there are conflicting Option values the last one overrides others.
-func (c *Client) EnqueueIn(d time.Duration, task *Task, opts ...Option) error {
+// By deafult, max retry is set to 25 and timeout is set to 30 minutes.
+func (c *Client) EnqueueIn(d time.Duration, task *Task, opts ...Option) (*Result, error) {
 	return c.enqueueAt(time.Now().Add(d), task, opts...)
 }
 
@@ -214,33 +267,55 @@ func (c *Client) Close() error {
 	return c.rdb.Close()
 }
 
-func (c *Client) enqueueAt(t time.Time, task *Task, opts ...Option) error {
+func (c *Client) enqueueAt(t time.Time, task *Task, opts ...Option) (*Result, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if defaults, ok := c.opts[task.Type]; ok {
 		opts = append(defaults, opts...)
 	}
 	opt := composeOptions(opts...)
+	deadline := noDeadline
+	if !opt.deadline.IsZero() {
+		deadline = opt.deadline
+	}
+	timeout := noTimeout
+	if opt.timeout != 0 {
+		timeout = opt.timeout
+	}
+	if deadline.Equal(noDeadline) && timeout == noTimeout {
+		// If neither deadline nor timeout are set, use default timeout.
+		timeout = defaultTimeout
+	}
 	msg := &base.TaskMessage{
-		ID:        xid.New(),
+		ID:        uuid.New(),
 		Type:      task.Type,
 		Payload:   task.Payload.data,
 		Queue:     opt.queue,
 		Retry:     opt.retry,
-		Timeout:   opt.timeout.String(),
-		Deadline:  opt.deadline.Format(time.RFC3339),
+		Deadline:  deadline.Unix(),
+		Timeout:   int64(timeout.Seconds()),
 		UniqueKey: uniqueKey(task, opt.uniqueTTL, opt.queue),
 	}
 	var err error
-	if time.Now().After(t) {
+	now := time.Now()
+	if t.Before(now) || t.Equal(now) {
 		err = c.enqueue(msg, opt.uniqueTTL)
 	} else {
 		err = c.schedule(msg, t, opt.uniqueTTL)
 	}
-	if err == rdb.ErrDuplicateTask {
-		return fmt.Errorf("%w", ErrDuplicateTask)
+	switch {
+	case err == rdb.ErrDuplicateTask:
+		return nil, fmt.Errorf("%w", ErrDuplicateTask)
+	case err != nil:
+		return nil, err
 	}
-	return err
+	return &Result{
+		ID:       msg.ID.String(),
+		Queue:    msg.Queue,
+		Retry:    msg.Retry,
+		Timeout:  timeout,
+		Deadline: deadline,
+	}, nil
 }
 
 func (c *Client) enqueue(msg *base.TaskMessage, uniqueTTL time.Duration) error {

client_test.go

@@ -15,11 +15,6 @@ import (
 	"github.com/hibiken/asynq/internal/base"
 )
 
-var (
-	noTimeout  = time.Duration(0).String()
-	noDeadline = time.Time{}.Format(time.RFC3339)
-)
-
 func TestClientEnqueueAt(t *testing.T) {
 	r := setup(t)
 	client := NewClient(RedisClientOpt{
@@ -39,14 +34,21 @@ func TestClientEnqueueAt(t *testing.T) {
 		task          *Task
 		processAt     time.Time
 		opts          []Option
+		wantRes       *Result
 		wantEnqueued  map[string][]*base.TaskMessage
-		wantScheduled []h.ZSetEntry
+		wantScheduled []base.Z
 	}{
 		{
 			desc:      "Process task immediately",
 			task:      task,
 			processAt: now,
 			opts:      []Option{},
+			wantRes: &Result{
+				Queue:    "default",
+				Retry:    defaultMaxRetry,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"default": {
 					{
@@ -54,30 +56,36 @@ func TestClientEnqueueAt(t *testing.T) {
 						Payload:  task.Payload.data,
 						Retry:    defaultMaxRetry,
 						Queue:    "default",
-						Timeout:  noTimeout,
-						Deadline: noDeadline,
+						Timeout:  int64(defaultTimeout.Seconds()),
+						Deadline: noDeadline.Unix(),
 					},
 				},
 			},
 			wantScheduled: nil, // db is flushed in setup so zset does not exist hence nil
 		},
 		{
-			desc:         "Schedule task to be processed in the future",
-			task:         task,
-			processAt:    oneHourLater,
-			opts:         []Option{},
+			desc:      "Schedule task to be processed in the future",
+			task:      task,
+			processAt: oneHourLater,
+			opts:      []Option{},
+			wantRes: &Result{
+				Queue:    "default",
+				Retry:    defaultMaxRetry,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
 			wantEnqueued: nil, // db is flushed in setup so list does not exist hence nil
-			wantScheduled: []h.ZSetEntry{
+			wantScheduled: []base.Z{
 				{
-					Msg: &base.TaskMessage{
+					Message: &base.TaskMessage{
 						Type:     task.Type,
 						Payload:  task.Payload.data,
 						Retry:    defaultMaxRetry,
 						Queue:    "default",
-						Timeout:  noTimeout,
-						Deadline: noDeadline,
+						Timeout:  int64(defaultTimeout.Seconds()),
+						Deadline: noDeadline.Unix(),
 					},
-					Score: float64(oneHourLater.Unix()),
+					Score: oneHourLater.Unix(),
 				},
 			},
 		},
@@ -86,11 +94,15 @@ func TestClientEnqueueAt(t *testing.T) {
 	for _, tc := range tests {
 		h.FlushDB(t, r) // clean up db before each test case.
 
-		err := client.EnqueueAt(tc.processAt, tc.task, tc.opts...)
+		gotRes, err := client.EnqueueAt(tc.processAt, tc.task, tc.opts...)
 		if err != nil {
 			t.Error(err)
 			continue
 		}
+		if diff := cmp.Diff(tc.wantRes, gotRes, cmpopts.IgnoreFields(Result{}, "ID")); diff != "" {
+			t.Errorf("%s;\nEnqueueAt(processAt, task) returned %v, want %v; (-want,+got)\n%s",
+				tc.desc, gotRes, tc.wantRes, diff)
+		}
 
 		for qname, want := range tc.wantEnqueued {
 			gotEnqueued := h.GetEnqueuedMessages(t, r, qname)
@@ -119,6 +131,7 @@ func TestClientEnqueue(t *testing.T) {
 		desc         string
 		task         *Task
 		opts         []Option
+		wantRes      *Result
 		wantEnqueued map[string][]*base.TaskMessage
 	}{
 		{
@@ -127,6 +140,12 @@ func TestClientEnqueue(t *testing.T) {
 			opts: []Option{
 				MaxRetry(3),
 			},
+			wantRes: &Result{
+				Queue:    "default",
+				Retry:    3,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"default": {
 					{
@@ -134,8 +153,8 @@ func TestClientEnqueue(t *testing.T) {
 						Payload:  task.Payload.data,
 						Retry:    3,
 						Queue:    "default",
-						Timeout:  noTimeout,
-						Deadline: noDeadline,
+						Timeout:  int64(defaultTimeout.Seconds()),
+						Deadline: noDeadline.Unix(),
 					},
 				},
 			},
@@ -146,6 +165,12 @@ func TestClientEnqueue(t *testing.T) {
 			opts: []Option{
 				MaxRetry(-2),
 			},
+			wantRes: &Result{
+				Queue:    "default",
+				Retry:    0,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"default": {
 					{
@@ -153,8 +178,8 @@ func TestClientEnqueue(t *testing.T) {
 						Payload:  task.Payload.data,
 						Retry:    0, // Retry count should be set to zero
 						Queue:    "default",
-						Timeout:  noTimeout,
-						Deadline: noDeadline,
+						Timeout:  int64(defaultTimeout.Seconds()),
+						Deadline: noDeadline.Unix(),
 					},
 				},
 			},
@@ -166,6 +191,12 @@ func TestClientEnqueue(t *testing.T) {
 				MaxRetry(2),
 				MaxRetry(10),
 			},
+			wantRes: &Result{
+				Queue:    "default",
+				Retry:    10,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"default": {
 					{
@@ -173,8 +204,8 @@ func TestClientEnqueue(t *testing.T) {
 						Payload:  task.Payload.data,
 						Retry:    10, // Last option takes precedence
 						Queue:    "default",
-						Timeout:  noTimeout,
-						Deadline: noDeadline,
+						Timeout:  int64(defaultTimeout.Seconds()),
+						Deadline: noDeadline.Unix(),
 					},
 				},
 			},
@@ -185,6 +216,12 @@ func TestClientEnqueue(t *testing.T) {
 			opts: []Option{
 				Queue("custom"),
 			},
+			wantRes: &Result{
+				Queue:    "custom",
+				Retry:    defaultMaxRetry,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"custom": {
 					{
@@ -192,8 +229,8 @@ func TestClientEnqueue(t *testing.T) {
 						Payload:  task.Payload.data,
 						Retry:    defaultMaxRetry,
 						Queue:    "custom",
-						Timeout:  noTimeout,
-						Deadline: noDeadline,
+						Timeout:  int64(defaultTimeout.Seconds()),
+						Deadline: noDeadline.Unix(),
 					},
 				},
 			},
@@ -204,6 +241,12 @@ func TestClientEnqueue(t *testing.T) {
 			opts: []Option{
 				Queue("HIGH"),
 			},
+			wantRes: &Result{
+				Queue:    "high",
+				Retry:    defaultMaxRetry,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"high": {
 					{
@@ -211,8 +254,8 @@ func TestClientEnqueue(t *testing.T) {
 						Payload:  task.Payload.data,
 						Retry:    defaultMaxRetry,
 						Queue:    "high",
-						Timeout:  noTimeout,
-						Deadline: noDeadline,
+						Timeout:  int64(defaultTimeout.Seconds()),
+						Deadline: noDeadline.Unix(),
 					},
 				},
 			},
@@ -223,6 +266,12 @@ func TestClientEnqueue(t *testing.T) {
 			opts: []Option{
 				Timeout(20 * time.Second),
 			},
+			wantRes: &Result{
+				Queue:    "default",
+				Retry:    defaultMaxRetry,
+				Timeout:  20 * time.Second,
+				Deadline: noDeadline,
+			},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"default": {
 					{
@@ -230,8 +279,8 @@ func TestClientEnqueue(t *testing.T) {
 						Payload:  task.Payload.data,
 						Retry:    defaultMaxRetry,
 						Queue:    "default",
-						Timeout:  (20 * time.Second).String(),
-						Deadline: noDeadline,
+						Timeout:  20,
+						Deadline: noDeadline.Unix(),
 					},
 				},
 			},
@@ -242,6 +291,12 @@ func TestClientEnqueue(t *testing.T) {
 			opts: []Option{
 				Deadline(time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC)),
 			},
+			wantRes: &Result{
+				Queue:    "default",
+				Retry:    defaultMaxRetry,
+				Timeout:  noTimeout,
+				Deadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC),
+			},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"default": {
 					{
@@ -249,8 +304,34 @@ func TestClientEnqueue(t *testing.T) {
 						Payload:  task.Payload.data,
 						Retry:    defaultMaxRetry,
 						Queue:    "default",
-						Timeout:  noTimeout,
-						Deadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+						Timeout:  int64(noTimeout.Seconds()),
+						Deadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC).Unix(),
+					},
+				},
+			},
+		},
+		{
+			desc: "With both deadline and timeout options",
+			task: task,
+			opts: []Option{
+				Timeout(20 * time.Second),
+				Deadline(time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC)),
+			},
+			wantRes: &Result{
+				Queue:    "default",
+				Retry:    defaultMaxRetry,
+				Timeout:  20 * time.Second,
+				Deadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC),
+			},
+			wantEnqueued: map[string][]*base.TaskMessage{
+				"default": {
+					{
+						Type:     task.Type,
+						Payload:  task.Payload.data,
+						Retry:    defaultMaxRetry,
+						Queue:    "default",
+						Timeout:  20,
+						Deadline: time.Date(2020, time.June, 24, 0, 0, 0, 0, time.UTC).Unix(),
 					},
 				},
 			},
@@ -260,11 +341,15 @@ func TestClientEnqueue(t *testing.T) {
 	for _, tc := range tests {
 		h.FlushDB(t, r) // clean up db before each test case.
 
-		err := client.Enqueue(tc.task, tc.opts...)
+		gotRes, err := client.Enqueue(tc.task, tc.opts...)
 		if err != nil {
 			t.Error(err)
 			continue
 		}
+		if diff := cmp.Diff(tc.wantRes, gotRes, cmpopts.IgnoreFields(Result{}, "ID")); diff != "" {
+			t.Errorf("%s;\nEnqueue(task) returned %v, want %v; (-want,+got)\n%s",
+				tc.desc, gotRes, tc.wantRes, diff)
+		}
 
 		for qname, want := range tc.wantEnqueued {
 			got := h.GetEnqueuedMessages(t, r, qname)
@@ -289,26 +374,33 @@ func TestClientEnqueueIn(t *testing.T) {
 		task          *Task
 		delay         time.Duration
 		opts          []Option
+		wantRes       *Result
 		wantEnqueued  map[string][]*base.TaskMessage
-		wantScheduled []h.ZSetEntry
+		wantScheduled []base.Z
 	}{
 		{
-			desc:         "schedule a task to be enqueued in one hour",
-			task:         task,
-			delay:        time.Hour,
-			opts:         []Option{},
+			desc:  "schedule a task to be enqueued in one hour",
+			task:  task,
+			delay: time.Hour,
+			opts:  []Option{},
+			wantRes: &Result{
+				Queue:    "default",
+				Retry:    defaultMaxRetry,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
 			wantEnqueued: nil, // db is flushed in setup so list does not exist hence nil
-			wantScheduled: []h.ZSetEntry{
+			wantScheduled: []base.Z{
 				{
-					Msg: &base.TaskMessage{
+					Message: &base.TaskMessage{
 						Type:     task.Type,
 						Payload:  task.Payload.data,
 						Retry:    defaultMaxRetry,
 						Queue:    "default",
-						Timeout:  noTimeout,
-						Deadline: noDeadline,
+						Timeout:  int64(defaultTimeout.Seconds()),
+						Deadline: noDeadline.Unix(),
 					},
-					Score: float64(time.Now().Add(time.Hour).Unix()),
+					Score: time.Now().Add(time.Hour).Unix(),
 				},
 			},
 		},
@@ -317,6 +409,12 @@ func TestClientEnqueueIn(t *testing.T) {
 			task:  task,
 			delay: 0,
 			opts:  []Option{},
+			wantRes: &Result{
+				Queue:    "default",
+				Retry:    defaultMaxRetry,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"default": {
 					{
@@ -324,8 +422,8 @@ func TestClientEnqueueIn(t *testing.T) {
 						Payload:  task.Payload.data,
 						Retry:    defaultMaxRetry,
 						Queue:    "default",
-						Timeout:  noTimeout,
-						Deadline: noDeadline,
+						Timeout:  int64(defaultTimeout.Seconds()),
+						Deadline: noDeadline.Unix(),
 					},
 				},
 			},
@@ -336,11 +434,15 @@ func TestClientEnqueueIn(t *testing.T) {
 	for _, tc := range tests {
 		h.FlushDB(t, r) // clean up db before each test case.
 
-		err := client.EnqueueIn(tc.delay, tc.task, tc.opts...)
+		gotRes, err := client.EnqueueIn(tc.delay, tc.task, tc.opts...)
 		if err != nil {
 			t.Error(err)
 			continue
 		}
+		if diff := cmp.Diff(tc.wantRes, gotRes, cmpopts.IgnoreFields(Result{}, "ID")); diff != "" {
+			t.Errorf("%s;\nEnqueueIn(delay, task) returned %v, want %v; (-want,+got)\n%s",
+				tc.desc, gotRes, tc.wantRes, diff)
+		}
 
 		for qname, want := range tc.wantEnqueued {
 			gotEnqueued := h.GetEnqueuedMessages(t, r, qname)
@@ -364,6 +466,7 @@ func TestClientDefaultOptions(t *testing.T) {
 		defaultOpts []Option // options set at the client level.
 		opts        []Option // options used at enqueue time.
 		task        *Task
+		wantRes     *Result
 		queue       string // queue that the message should go into.
 		want        *base.TaskMessage
 	}{
@@ -372,14 +475,20 @@ func TestClientDefaultOptions(t *testing.T) {
 			defaultOpts: []Option{Queue("feed")},
 			opts:        []Option{},
 			task:        NewTask("feed:import", nil),
-			queue:       "feed",
+			wantRes: &Result{
+				Queue:    "feed",
+				Retry:    defaultMaxRetry,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
+			queue: "feed",
 			want: &base.TaskMessage{
 				Type:     "feed:import",
 				Payload:  nil,
 				Retry:    defaultMaxRetry,
 				Queue:    "feed",
-				Timeout:  noTimeout,
-				Deadline: noDeadline,
+				Timeout:  int64(defaultTimeout.Seconds()),
+				Deadline: noDeadline.Unix(),
 			},
 		},
 		{
@@ -387,14 +496,20 @@ func TestClientDefaultOptions(t *testing.T) {
 			defaultOpts: []Option{Queue("feed"), MaxRetry(5)},
 			opts:        []Option{},
 			task:        NewTask("feed:import", nil),
-			queue:       "feed",
+			wantRes: &Result{
+				Queue:    "feed",
+				Retry:    5,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
+			queue: "feed",
 			want: &base.TaskMessage{
 				Type:     "feed:import",
 				Payload:  nil,
 				Retry:    5,
 				Queue:    "feed",
-				Timeout:  noTimeout,
-				Deadline: noDeadline,
+				Timeout:  int64(defaultTimeout.Seconds()),
+				Deadline: noDeadline.Unix(),
 			},
 		},
 		{
@@ -402,14 +517,20 @@ func TestClientDefaultOptions(t *testing.T) {
 			defaultOpts: []Option{Queue("feed"), MaxRetry(5)},
 			opts:        []Option{Queue("critical")},
 			task:        NewTask("feed:import", nil),
-			queue:       "critical",
+			wantRes: &Result{
+				Queue:    "critical",
+				Retry:    5,
+				Timeout:  defaultTimeout,
+				Deadline: noDeadline,
+			},
+			queue: "critical",
 			want: &base.TaskMessage{
 				Type:     "feed:import",
 				Payload:  nil,
 				Retry:    5,
 				Queue:    "critical",
-				Timeout:  noTimeout,
-				Deadline: noDeadline,
+				Timeout:  int64(defaultTimeout.Seconds()),
+				Deadline: noDeadline.Unix(),
 			},
 		},
 	}
@@ -418,10 +539,14 @@ func TestClientDefaultOptions(t *testing.T) {
 		h.FlushDB(t, r)
 		c := NewClient(RedisClientOpt{Addr: redisAddr, DB: redisDB})
 		c.SetDefaultOptions(tc.task.Type, tc.defaultOpts...)
-		err := c.Enqueue(tc.task, tc.opts...)
+		gotRes, err := c.Enqueue(tc.task, tc.opts...)
 		if err != nil {
 			t.Fatal(err)
 		}
+		if diff := cmp.Diff(tc.wantRes, gotRes, cmpopts.IgnoreFields(Result{}, "ID")); diff != "" {
+			t.Errorf("%s;\nEnqueue(task, opts...) returned %v, want %v; (-want,+got)\n%s",
+				tc.desc, gotRes, tc.wantRes, diff)
+		}
 		enqueued := h.GetEnqueuedMessages(t, r, tc.queue)
 		if len(enqueued) != 1 {
 			t.Errorf("%s;\nexpected queue %q to have one message; got %d messages in the queue.",
@@ -523,7 +648,7 @@ func TestEnqueueUnique(t *testing.T) {
 		h.FlushDB(t, r) // clean up db before each test case.
 
 		// Enqueue the task first. It should succeed.
-		err := c.Enqueue(tc.task, Unique(tc.ttl))
+		_, err := c.Enqueue(tc.task, Unique(tc.ttl))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -535,7 +660,7 @@ func TestEnqueueUnique(t *testing.T) {
 		}
 
 		// Enqueue the task again. It should fail.
-		err = c.Enqueue(tc.task, Unique(tc.ttl))
+		_, err = c.Enqueue(tc.task, Unique(tc.ttl))
 		if err == nil {
 			t.Errorf("Enqueueing %+v did not return an error", tc.task)
 			continue
@@ -570,7 +695,7 @@ func TestEnqueueInUnique(t *testing.T) {
 		h.FlushDB(t, r) // clean up db before each test case.
 
 		// Enqueue the task first. It should succeed.
-		err := c.EnqueueIn(tc.d, tc.task, Unique(tc.ttl))
+		_, err := c.EnqueueIn(tc.d, tc.task, Unique(tc.ttl))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -583,7 +708,7 @@ func TestEnqueueInUnique(t *testing.T) {
 		}
 
 		// Enqueue the task again. It should fail.
-		err = c.EnqueueIn(tc.d, tc.task, Unique(tc.ttl))
+		_, err = c.EnqueueIn(tc.d, tc.task, Unique(tc.ttl))
 		if err == nil {
 			t.Errorf("Enqueueing %+v did not return an error", tc.task)
 			continue
@@ -618,7 +743,7 @@ func TestEnqueueAtUnique(t *testing.T) {
 		h.FlushDB(t, r) // clean up db before each test case.
 
 		// Enqueue the task first. It should succeed.
-		err := c.EnqueueAt(tc.at, tc.task, Unique(tc.ttl))
+		_, err := c.EnqueueAt(tc.at, tc.task, Unique(tc.ttl))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -631,7 +756,7 @@ func TestEnqueueAtUnique(t *testing.T) {
 		}
 
 		// Enqueue the task again. It should fail.
-		err = c.EnqueueAt(tc.at, tc.task, Unique(tc.ttl))
+		_, err = c.EnqueueAt(tc.at, tc.task, Unique(tc.ttl))
 		if err == nil {
 			t.Errorf("Enqueueing %+v did not return an error", tc.task)
 			continue

context.go

@@ -27,25 +27,14 @@ type ctxKey int
 const metadataCtxKey ctxKey = 0
 
 // createContext returns a context and cancel function for a given task message.
-func createContext(msg *base.TaskMessage) (ctx context.Context, cancel context.CancelFunc) {
+func createContext(msg *base.TaskMessage, deadline time.Time) (context.Context, context.CancelFunc) {
 	metadata := taskMetadata{
 		id:         msg.ID.String(),
 		maxRetry:   msg.Retry,
 		retryCount: msg.Retried,
 	}
-	ctx = context.WithValue(context.Background(), metadataCtxKey, metadata)
-	timeout, err := time.ParseDuration(msg.Timeout)
-	if err == nil && timeout != 0 {
-		ctx, cancel = context.WithTimeout(ctx, timeout)
-	}
-	deadline, err := time.Parse(time.RFC3339, msg.Deadline)
-	if err == nil && !deadline.IsZero() {
-		ctx, cancel = context.WithDeadline(ctx, deadline)
-	}
-	if cancel == nil {
-		ctx, cancel = context.WithCancel(ctx)
-	}
-	return ctx, cancel
+	ctx := context.WithValue(context.Background(), metadataCtxKey, metadata)
+	return context.WithDeadline(ctx, deadline)
 }
 
 // GetTaskID extracts a task ID from a context, if any.

context_test.go

@@ -10,51 +10,38 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
 	"github.com/hibiken/asynq/internal/base"
-	"github.com/rs/xid"
 )
 
-func TestCreateContextWithTimeRestrictions(t *testing.T) {
-	var (
-		noTimeout  = time.Duration(0)
-		noDeadline = time.Time{}
-	)
-
+func TestCreateContextWithFutureDeadline(t *testing.T) {
 	tests := []struct {
-		desc         string
-		timeout      time.Duration
-		deadline     time.Time
-		wantDeadline time.Time
+		deadline time.Time
 	}{
-		{"only with timeout", 10 * time.Second, noDeadline, time.Now().Add(10 * time.Second)},
-		{"only with deadline", noTimeout, time.Now().Add(time.Hour), time.Now().Add(time.Hour)},
-		{"with timeout and deadline (timeout < deadline)", 10 * time.Second, time.Now().Add(time.Hour), time.Now().Add(10 * time.Second)},
-		{"with timeout and deadline (timeout > deadline)", 10 * time.Minute, time.Now().Add(30 * time.Second), time.Now().Add(30 * time.Second)},
+		{time.Now().Add(time.Hour)},
 	}
 
 	for _, tc := range tests {
 		msg := &base.TaskMessage{
-			Type:     "something",
-			ID:       xid.New(),
-			Timeout:  tc.timeout.String(),
-			Deadline: tc.deadline.Format(time.RFC3339),
+			Type:    "something",
+			ID:      uuid.New(),
+			Payload: nil,
 		}
 
-		ctx, cancel := createContext(msg)
+		ctx, cancel := createContext(msg, tc.deadline)
 
 		select {
 		case x := <-ctx.Done():
-			t.Errorf("%s: <-ctx.Done() == %v, want nothing (it should block)", tc.desc, x)
+			t.Errorf("<-ctx.Done() == %v, want nothing (it should block)", x)
 		default:
 		}
 
 		got, ok := ctx.Deadline()
 		if !ok {
-			t.Errorf("%s: ctx.Deadline() returned false, want deadline to be set", tc.desc)
+			t.Errorf("ctx.Deadline() returned false, want deadline to be set")
 		}
-		if !cmp.Equal(tc.wantDeadline, got, cmpopts.EquateApproxTime(time.Second)) {
-			t.Errorf("%s: ctx.Deadline() returned %v, want %v", tc.desc, got, tc.wantDeadline)
+		if !cmp.Equal(tc.deadline, got) {
+			t.Errorf("ctx.Deadline() returned %v, want %v", got, tc.deadline)
 		}
 
 		cancel()
@@ -67,33 +54,36 @@ func TestCreateContextWithTimeRestrictions(t *testing.T) {
 	}
 }
 
-func TestCreateContextWithoutTimeRestrictions(t *testing.T) {
-	msg := &base.TaskMessage{
-		Type:     "something",
-		ID:       xid.New(),
-		Timeout:  time.Duration(0).String(),        // zero value to indicate no timeout
-		Deadline: time.Time{}.Format(time.RFC3339), // zero value to indicate no deadline
+func TestCreateContextWithPastDeadline(t *testing.T) {
+	tests := []struct {
+		deadline time.Time
+	}{
+		{time.Now().Add(-2 * time.Hour)},
 	}
 
-	ctx, cancel := createContext(msg)
+	for _, tc := range tests {
+		msg := &base.TaskMessage{
+			Type:    "something",
+			ID:      uuid.New(),
+			Payload: nil,
+		}
 
-	select {
-	case x := <-ctx.Done():
-		t.Errorf("<-ctx.Done() == %v, want nothing (it should block)", x)
-	default:
-	}
+		ctx, cancel := createContext(msg, tc.deadline)
+		defer cancel()
 
-	_, ok := ctx.Deadline()
-	if ok {
-		t.Error("ctx.Deadline() returned true, want deadline to not be set")
-	}
+		select {
+		case <-ctx.Done():
+		default:
+			t.Errorf("ctx.Done() blocked, want it to be non-blocking")
+		}
 
-	cancel()
-
-	select {
-	case <-ctx.Done():
-	default:
-		t.Error("ctx.Done() blocked, want it to be non-blocking")
+		got, ok := ctx.Deadline()
+		if !ok {
+			t.Errorf("ctx.Deadline() returned false, want deadline to be set")
+		}
+		if !cmp.Equal(tc.deadline, got) {
+			t.Errorf("ctx.Deadline() returned %v, want %v", got, tc.deadline)
+		}
 	}
 }
 
@@ -102,12 +92,13 @@ func TestGetTaskMetadataFromContext(t *testing.T) {
 		desc string
 		msg  *base.TaskMessage
 	}{
-		{"with zero retried message", &base.TaskMessage{Type: "something", ID: xid.New(), Retry: 25, Retried: 0}},
-		{"with non-zero retried message", &base.TaskMessage{Type: "something", ID: xid.New(), Retry: 10, Retried: 5}},
+		{"with zero retried message", &base.TaskMessage{Type: "something", ID: uuid.New(), Retry: 25, Retried: 0, Timeout: 1800}},
+		{"with non-zero retried message", &base.TaskMessage{Type: "something", ID: uuid.New(), Retry: 10, Retried: 5, Timeout: 1800}},
 	}
 
 	for _, tc := range tests {
-		ctx, _ := createContext(tc.msg)
+		ctx, cancel := createContext(tc.msg, time.Now().Add(30*time.Minute))
+		defer cancel()
 
 		id, ok := GetTaskID(ctx)
 		if !ok {

doc.go

@@ -25,10 +25,10 @@ Task is created with two parameters: its type and payload.
         map[string]interface{}{"user_id": 42})
 
     // Enqueue the task to be processed immediately.
-    err := client.Enqueue(t)
+    res, err := client.Enqueue(t)
 
     // Schedule the task to be processed after one minute.
-    err = client.EnqueueIn(time.Minute, t)
+    res, err = client.EnqueueIn(time.Minute, t)
 
 The Server is used to run the background task processing with a given
 handler.

go.mod

@@ -5,7 +5,7 @@ go 1.13
 require (
 	github.com/go-redis/redis/v7 v7.2.0
 	github.com/google/go-cmp v0.4.0
-	github.com/rs/xid v1.2.1
+	github.com/google/uuid v1.1.1
 	github.com/spf13/cast v1.3.1
 	go.uber.org/goleak v0.10.0
 	golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e

go.sum

@@ -2,16 +2,15 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/go-redis/redis/v7 v7.0.0-beta.4 h1:p6z7Pde69EGRWvlC++y8aFcaWegyrKHzOBGo0zUACTQ=
-github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
 github.com/go-redis/redis/v7 v7.2.0 h1:CrCexy/jYWZjW0AyVoHlcJUeZN19VWlbepTh1Vq6dJs=
 github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -20,16 +19,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
-github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
@@ -39,10 +34,8 @@ go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -60,8 +53,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

healthcheck.go

@@ -0,0 +1,80 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package asynq
+
+import (
+	"sync"
+	"time"
+
+	"github.com/hibiken/asynq/internal/base"
+	"github.com/hibiken/asynq/internal/log"
+)
+
+// healthchecker is responsible for pinging broker periodically
+// and call user provided HeathCheckFunc with the ping result.
+type healthchecker struct {
+	logger *log.Logger
+	broker base.Broker
+
+	// channel to communicate back to the long running "healthchecker" goroutine.
+	done chan struct{}
+
+	// interval between healthchecks.
+	interval time.Duration
+
+	// function to call periodically.
+	healthcheckFunc func(error)
+}
+
+type healthcheckerParams struct {
+	logger          *log.Logger
+	broker          base.Broker
+	interval        time.Duration
+	healthcheckFunc func(error)
+}
+
+func newHealthChecker(params healthcheckerParams) *healthchecker {
+	return &healthchecker{
+		logger:          params.logger,
+		broker:          params.broker,
+		done:            make(chan struct{}),
+		interval:        params.interval,
+		healthcheckFunc: params.healthcheckFunc,
+	}
+}
+
+func (hc *healthchecker) terminate() {
+	if hc.healthcheckFunc == nil {
+		return
+	}
+
+	hc.logger.Debug("Healthchecker shutting down...")
+	// Signal the healthchecker goroutine to stop.
+	hc.done <- struct{}{}
+}
+
+func (hc *healthchecker) start(wg *sync.WaitGroup) {
+	if hc.healthcheckFunc == nil {
+		return
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		timer := time.NewTimer(hc.interval)
+		for {
+			select {
+			case <-hc.done:
+				hc.logger.Debug("Healthchecker done")
+				timer.Stop()
+				return
+			case <-timer.C:
+				err := hc.broker.Ping()
+				hc.healthcheckFunc(err)
+				timer.Reset(hc.interval)
+			}
+		}
+	}()
+}

healthcheck_test.go

@@ -0,0 +1,101 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package asynq
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/hibiken/asynq/internal/testbroker"
+)
+
+func TestHealthChecker(t *testing.T) {
+	r := setup(t)
+	rdbClient := rdb.NewRDB(r)
+
+	var (
+		// mu guards called and e variables.
+		mu     sync.Mutex
+		called int
+		e      error
+	)
+	checkFn := func(err error) {
+		mu.Lock()
+		defer mu.Unlock()
+		called++
+		e = err
+	}
+
+	hc := newHealthChecker(healthcheckerParams{
+		logger:          testLogger,
+		broker:          rdbClient,
+		interval:        1 * time.Second,
+		healthcheckFunc: checkFn,
+	})
+
+	hc.start(&sync.WaitGroup{})
+
+	time.Sleep(2 * time.Second)
+
+	mu.Lock()
+	if called == 0 {
+		t.Errorf("Healthchecker did not call the provided HealthCheckFunc")
+	}
+	if e != nil {
+		t.Errorf("HealthCheckFunc was called with non-nil error: %v", e)
+	}
+	mu.Unlock()
+
+	hc.terminate()
+}
+
+func TestHealthCheckerWhenRedisDown(t *testing.T) {
+	// Make sure that healthchecker goroutine doesn't panic
+	// if it cannot connect to redis.
+	defer func() {
+		if r := recover(); r != nil {
+			t.Errorf("panic occurred: %v", r)
+		}
+	}()
+	r := rdb.NewRDB(setup(t))
+	testBroker := testbroker.NewTestBroker(r)
+	var (
+		// mu guards called and e variables.
+		mu     sync.Mutex
+		called int
+		e      error
+	)
+	checkFn := func(err error) {
+		mu.Lock()
+		defer mu.Unlock()
+		called++
+		e = err
+	}
+
+	hc := newHealthChecker(healthcheckerParams{
+		logger:          testLogger,
+		broker:          testBroker,
+		interval:        1 * time.Second,
+		healthcheckFunc: checkFn,
+	})
+
+	testBroker.Sleep()
+	hc.start(&sync.WaitGroup{})
+
+	time.Sleep(2 * time.Second)
+
+	mu.Lock()
+	if called == 0 {
+		t.Errorf("Healthchecker did not call the provided HealthCheckFunc")
+	}
+	if e == nil {
+		t.Errorf("HealthCheckFunc was called with nil; want non-nil error")
+	}
+	mu.Unlock()
+
+	hc.terminate()
+}

heartbeat.go

@@ -5,9 +5,11 @@
 package asynq
 
 import (
+	"os"
 	"sync"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/hibiken/asynq/internal/base"
 	"github.com/hibiken/asynq/internal/log"
 )
@@ -18,29 +20,69 @@ type heartbeater struct {
 	logger *log.Logger
 	broker base.Broker
 
-	ss *base.ServerState
-
 	// channel to communicate back to the long running "heartbeater" goroutine.
 	done chan struct{}
 
 	// interval between heartbeats.
 	interval time.Duration
+
+	// following fields are initialized at construction time and are immutable.
+	host           string
+	pid            int
+	serverID       string
+	concurrency    int
+	queues         map[string]int
+	strictPriority bool
+
+	// following fields are mutable and should be accessed only by the
+	// heartbeater goroutine. In other words, confine these variables
+	// to this goroutine only.
+	started time.Time
+	workers map[string]workerStat
+
+	// status is shared with other goroutine but is concurrency safe.
+	status *base.ServerStatus
+
+	// channels to receive updates on active workers.
+	starting <-chan *base.TaskMessage
+	finished <-chan *base.TaskMessage
 }
 
 type heartbeaterParams struct {
-	logger      *log.Logger
-	broker      base.Broker
-	serverState *base.ServerState
-	interval    time.Duration
+	logger         *log.Logger
+	broker         base.Broker
+	interval       time.Duration
+	concurrency    int
+	queues         map[string]int
+	strictPriority bool
+	status         *base.ServerStatus
+	starting       <-chan *base.TaskMessage
+	finished       <-chan *base.TaskMessage
 }
 
 func newHeartbeater(params heartbeaterParams) *heartbeater {
+	host, err := os.Hostname()
+	if err != nil {
+		host = "unknown-host"
+	}
+
 	return &heartbeater{
 		logger:   params.logger,
 		broker:   params.broker,
-		ss:       params.serverState,
 		done:     make(chan struct{}),
 		interval: params.interval,
+
+		host:           host,
+		pid:            os.Getpid(),
+		serverID:       uuid.New().String(),
+		concurrency:    params.concurrency,
+		queues:         params.queues,
+		strictPriority: params.strictPriority,
+
+		status:   params.status,
+		workers:  make(map[string]workerStat),
+		starting: params.starting,
+		finished: params.finished,
 	}
 }
 
@@ -50,31 +92,74 @@ func (h *heartbeater) terminate() {
 	h.done <- struct{}{}
 }
 
+// A workerStat records the message a worker is working on
+// and the time the worker has started processing the message.
+type workerStat struct {
+	started time.Time
+	msg     *base.TaskMessage
+}
+
 func (h *heartbeater) start(wg *sync.WaitGroup) {
-	h.ss.SetStarted(time.Now())
-	h.ss.SetStatus(base.StatusRunning)
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
+
+		h.started = time.Now()
+
 		h.beat()
+
+		timer := time.NewTimer(h.interval)
 		for {
 			select {
 			case <-h.done:
-				h.broker.ClearServerState(h.ss)
+				h.broker.ClearServerState(h.host, h.pid, h.serverID)
 				h.logger.Debug("Heartbeater done")
+				timer.Stop()
 				return
-			case <-time.After(h.interval):
+
+			case <-timer.C:
 				h.beat()
+				timer.Reset(h.interval)
+
+			case msg := <-h.starting:
+				h.workers[msg.ID.String()] = workerStat{time.Now(), msg}
+
+			case msg := <-h.finished:
+				delete(h.workers, msg.ID.String())
 			}
 		}
 	}()
 }
 
 func (h *heartbeater) beat() {
+	info := base.ServerInfo{
+		Host:              h.host,
+		PID:               h.pid,
+		ServerID:          h.serverID,
+		Concurrency:       h.concurrency,
+		Queues:            h.queues,
+		StrictPriority:    h.strictPriority,
+		Status:            h.status.String(),
+		Started:           h.started,
+		ActiveWorkerCount: len(h.workers),
+	}
+
+	var ws []*base.WorkerInfo
+	for id, stat := range h.workers {
+		ws = append(ws, &base.WorkerInfo{
+			Host:    h.host,
+			PID:     h.pid,
+			ID:      id,
+			Type:    stat.msg.Type,
+			Queue:   stat.msg.Queue,
+			Payload: stat.msg.Payload,
+			Started: stat.started,
+		})
+	}
+
 	// Note: Set TTL to be long enough so that it won't expire before we write again
 	// and short enough to expire quickly once the process is shut down or killed.
-	err := h.broker.WriteServerState(h.ss, h.interval*2)
-	if err != nil {
-		h.logger.Errorf("could not write heartbeat data: %v", err)
+	if err := h.broker.WriteServerState(&info, ws, h.interval*2); err != nil {
+		h.logger.Errorf("could not write server state data: %v", err)
 	}
 }

heartbeat_test.go

@@ -37,14 +37,24 @@ func TestHeartbeater(t *testing.T) {
 	for _, tc := range tests {
 		h.FlushDB(t, r)
 
-		state := base.NewServerState(tc.host, tc.pid, tc.concurrency, tc.queues, false)
+		status := base.NewServerStatus(base.StatusIdle)
 		hb := newHeartbeater(heartbeaterParams{
-			logger:      testLogger,
-			broker:      rdbClient,
-			serverState: state,
-			interval:    tc.interval,
+			logger:         testLogger,
+			broker:         rdbClient,
+			interval:       tc.interval,
+			concurrency:    tc.concurrency,
+			queues:         tc.queues,
+			strictPriority: false,
+			status:         status,
+			starting:       make(chan *base.TaskMessage),
+			finished:       make(chan *base.TaskMessage),
 		})
 
+		// Change host and pid fields for testing purpose.
+		hb.host = tc.host
+		hb.pid = tc.pid
+
+		status.Set(base.StatusRunning)
 		var wg sync.WaitGroup
 		hb.start(&wg)
 
@@ -80,7 +90,7 @@ func TestHeartbeater(t *testing.T) {
 		}
 
 		// status change
-		state.SetStatus(base.StatusStopped)
+		status.Set(base.StatusStopped)
 
 		// allow for heartbeater to write to redis
 		time.Sleep(tc.interval * 2)
@@ -119,12 +129,16 @@ func TestHeartbeaterWithRedisDown(t *testing.T) {
 	}()
 	r := rdb.NewRDB(setup(t))
 	testBroker := testbroker.NewTestBroker(r)
-	ss := base.NewServerState("localhost", 1234, 10, map[string]int{"default": 1}, false)
 	hb := newHeartbeater(heartbeaterParams{
-		logger:      testLogger,
-		broker:      testBroker,
-		serverState: ss,
-		interval:    time.Second,
+		logger:         testLogger,
+		broker:         testBroker,
+		interval:       time.Second,
+		concurrency:    10,
+		queues:         map[string]int{"default": 1},
+		strictPriority: false,
+		status:         base.NewServerStatus(base.StatusRunning),
+		starting:       make(chan *base.TaskMessage),
+		finished:       make(chan *base.TaskMessage),
 	})
 
 	testBroker.Sleep()

inspector.go

@@ -0,0 +1,500 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package asynq
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/hibiken/asynq/internal/rdb"
+)
+
+// Inspector is a client interface to inspect and mutate the state of
+// queues and tasks.
+type Inspector struct {
+	rdb *rdb.RDB
+}
+
+// New returns a new instance of Inspector.
+func NewInspector(r RedisConnOpt) *Inspector {
+	return &Inspector{
+		rdb: rdb.NewRDB(createRedisClient(r)),
+	}
+}
+
+// Stats represents a state of queues at a certain time.
+type Stats struct {
+	Enqueued   int
+	InProgress int
+	Scheduled  int
+	Retry      int
+	Dead       int
+	Processed  int
+	Failed     int
+	Queues     []*QueueInfo
+	Timestamp  time.Time
+}
+
+// QueueInfo holds information about a queue.
+type QueueInfo struct {
+	// Name of the queue (e.g. "default", "critical").
+	// Note: It doesn't include the prefix "asynq:queues:".
+	Name string
+
+	// Paused indicates whether the queue is paused.
+	// If true, tasks in the queue should not be processed.
+	Paused bool
+
+	// Size is the number of tasks in the queue.
+	Size int
+}
+
+// CurrentStats returns a current stats of the queues.
+func (i *Inspector) CurrentStats() (*Stats, error) {
+	stats, err := i.rdb.CurrentStats()
+	if err != nil {
+		return nil, err
+	}
+	var qs []*QueueInfo
+	for _, q := range stats.Queues {
+		qs = append(qs, (*QueueInfo)(q))
+	}
+	return &Stats{
+		Enqueued:   stats.Enqueued,
+		InProgress: stats.InProgress,
+		Scheduled:  stats.Scheduled,
+		Retry:      stats.Retry,
+		Dead:       stats.Dead,
+		Processed:  stats.Processed,
+		Failed:     stats.Failed,
+		Queues:     qs,
+		Timestamp:  stats.Timestamp,
+	}, nil
+}
+
+// DailyStats holds aggregate data for a given day.
+type DailyStats struct {
+	Processed int
+	Failed    int
+	Date      time.Time
+}
+
+// History returns a list of stats from the last n days.
+func (i *Inspector) History(n int) ([]*DailyStats, error) {
+	stats, err := i.rdb.HistoricalStats(n)
+	if err != nil {
+		return nil, err
+	}
+	var res []*DailyStats
+	for _, s := range stats {
+		res = append(res, &DailyStats{
+			Processed: s.Processed,
+			Failed:    s.Failed,
+			Date:      s.Time,
+		})
+	}
+	return res, nil
+}
+
+// EnqueuedTask is a task in a queue and is ready to be processed.
+type EnqueuedTask struct {
+	*Task
+	ID    string
+	Queue string
+}
+
+// InProgressTask is a task that's currently being processed.
+type InProgressTask struct {
+	*Task
+	ID string
+}
+
+// ScheduledTask is a task scheduled to be processed in the future.
+type ScheduledTask struct {
+	*Task
+	ID            string
+	Queue         string
+	NextEnqueueAt time.Time
+
+	score int64
+}
+
+// RetryTask is a task scheduled to be retried in the future.
+type RetryTask struct {
+	*Task
+	ID            string
+	Queue         string
+	NextEnqueueAt time.Time
+	MaxRetry      int
+	Retried       int
+	ErrorMsg      string
+	// TODO: LastFailedAt  time.Time
+
+	score int64
+}
+
+// DeadTask is a task exhausted its retries.
+// DeadTask won't be retried automatically.
+type DeadTask struct {
+	*Task
+	ID           string
+	Queue        string
+	MaxRetry     int
+	Retried      int
+	LastFailedAt time.Time
+	ErrorMsg     string
+
+	score int64
+}
+
+// Key returns a key used to delete, enqueue, and kill the task.
+func (t *ScheduledTask) Key() string {
+	return fmt.Sprintf("s:%v:%v", t.ID, t.score)
+}
+
+// Key returns a key used to delete, enqueue, and kill the task.
+func (t *RetryTask) Key() string {
+	return fmt.Sprintf("r:%v:%v", t.ID, t.score)
+}
+
+// Key returns a key used to delete, enqueue, and kill the task.
+func (t *DeadTask) Key() string {
+	return fmt.Sprintf("d:%v:%v", t.ID, t.score)
+}
+
+// parseTaskKey parses a key string and returns each part of key with proper
+// type if valid, otherwise it reports an error.
+func parseTaskKey(key string) (id uuid.UUID, score int64, qtype string, err error) {
+	parts := strings.Split(key, ":")
+	if len(parts) != 3 {
+		return uuid.Nil, 0, "", fmt.Errorf("invalid id")
+	}
+	id, err = uuid.Parse(parts[1])
+	if err != nil {
+		return uuid.Nil, 0, "", fmt.Errorf("invalid id")
+	}
+	score, err = strconv.ParseInt(parts[2], 10, 64)
+	if err != nil {
+		return uuid.Nil, 0, "", fmt.Errorf("invalid id")
+	}
+	qtype = parts[0]
+	if len(qtype) != 1 || !strings.Contains("srd", qtype) {
+		return uuid.Nil, 0, "", fmt.Errorf("invalid id")
+	}
+	return id, score, qtype, nil
+}
+
+// ListOption specifies behavior of list operation.
+type ListOption interface{}
+
+// Internal list option representations.
+type (
+	pageSizeOpt int
+	pageNumOpt  int
+)
+
+type listOption struct {
+	pageSize int
+	pageNum  int
+}
+
+const (
+	// Page size used by default in list operation.
+	defaultPageSize = 30
+
+	// Page number used by default in list operation.
+	defaultPageNum = 1
+)
+
+func composeListOptions(opts ...ListOption) listOption {
+	res := listOption{
+		pageSize: defaultPageSize,
+		pageNum:  defaultPageNum,
+	}
+	for _, opt := range opts {
+		switch opt := opt.(type) {
+		case pageSizeOpt:
+			res.pageSize = int(opt)
+		case pageNumOpt:
+			res.pageNum = int(opt)
+		default:
+			// ignore unexpected option
+		}
+	}
+	return res
+}
+
+// PageSize returns an option to specify the page size for list operation.
+//
+// Negative page size is treated as zero.
+func PageSize(n int) ListOption {
+	if n < 0 {
+		n = 0
+	}
+	return pageSizeOpt(n)
+}
+
+// Page returns an option to specify the page number for list operation.
+// The value 1 fetches the first page.
+//
+// Negative page number is treated as one.
+func Page(n int) ListOption {
+	if n < 0 {
+		n = 1
+	}
+	return pageNumOpt(n)
+}
+
+// ListScheduledTasks retrieves tasks in the specified queue.
+//
+// By default, it retrieves the first 30 tasks.
+func (i *Inspector) ListEnqueuedTasks(qname string, opts ...ListOption) ([]*EnqueuedTask, error) {
+	opt := composeListOptions(opts...)
+	pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}
+	msgs, err := i.rdb.ListEnqueued(qname, pgn)
+	if err != nil {
+		return nil, err
+	}
+	var tasks []*EnqueuedTask
+	for _, m := range msgs {
+		tasks = append(tasks, &EnqueuedTask{
+			Task:  NewTask(m.Type, m.Payload),
+			ID:    m.ID.String(),
+			Queue: m.Queue,
+		})
+	}
+	return tasks, err
+}
+
+// ListScheduledTasks retrieves tasks currently being processed.
+//
+// By default, it retrieves the first 30 tasks.
+func (i *Inspector) ListInProgressTasks(opts ...ListOption) ([]*InProgressTask, error) {
+	opt := composeListOptions(opts...)
+	pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}
+	msgs, err := i.rdb.ListInProgress(pgn)
+	if err != nil {
+		return nil, err
+	}
+	var tasks []*InProgressTask
+	for _, m := range msgs {
+		tasks = append(tasks, &InProgressTask{
+			Task: NewTask(m.Type, m.Payload),
+			ID:   m.ID.String(),
+		})
+	}
+	return tasks, err
+}
+
+// ListScheduledTasks retrieves tasks in scheduled state.
+// Tasks are sorted by NextEnqueueAt field in ascending order.
+//
+// By default, it retrieves the first 30 tasks.
+func (i *Inspector) ListScheduledTasks(opts ...ListOption) ([]*ScheduledTask, error) {
+	opt := composeListOptions(opts...)
+	pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}
+	zs, err := i.rdb.ListScheduled(pgn)
+	if err != nil {
+		return nil, err
+	}
+	var tasks []*ScheduledTask
+	for _, z := range zs {
+		enqueueAt := time.Unix(z.Score, 0)
+		t := NewTask(z.Message.Type, z.Message.Payload)
+		tasks = append(tasks, &ScheduledTask{
+			Task:          t,
+			ID:            z.Message.ID.String(),
+			Queue:         z.Message.Queue,
+			NextEnqueueAt: enqueueAt,
+			score:         z.Score,
+		})
+	}
+	return tasks, nil
+}
+
+// ListScheduledTasks retrieves tasks in retry state.
+// Tasks are sorted by NextEnqueueAt field in ascending order.
+//
+// By default, it retrieves the first 30 tasks.
+func (i *Inspector) ListRetryTasks(opts ...ListOption) ([]*RetryTask, error) {
+	opt := composeListOptions(opts...)
+	pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}
+	zs, err := i.rdb.ListRetry(pgn)
+	if err != nil {
+		return nil, err
+	}
+	var tasks []*RetryTask
+	for _, z := range zs {
+		enqueueAt := time.Unix(z.Score, 0)
+		t := NewTask(z.Message.Type, z.Message.Payload)
+		tasks = append(tasks, &RetryTask{
+			Task:          t,
+			ID:            z.Message.ID.String(),
+			Queue:         z.Message.Queue,
+			NextEnqueueAt: enqueueAt,
+			MaxRetry:      z.Message.Retry,
+			Retried:       z.Message.Retried,
+			// TODO: LastFailedAt: z.Message.LastFailedAt
+			ErrorMsg: z.Message.ErrorMsg,
+			score:    z.Score,
+		})
+	}
+	return tasks, nil
+}
+
+// ListScheduledTasks retrieves tasks in retry state.
+// Tasks are sorted by LastFailedAt field in descending order.
+//
+// By default, it retrieves the first 30 tasks.
+func (i *Inspector) ListDeadTasks(opts ...ListOption) ([]*DeadTask, error) {
+	opt := composeListOptions(opts...)
+	pgn := rdb.Pagination{Size: opt.pageSize, Page: opt.pageNum - 1}
+	zs, err := i.rdb.ListDead(pgn)
+	if err != nil {
+		return nil, err
+	}
+	var tasks []*DeadTask
+	for _, z := range zs {
+		failedAt := time.Unix(z.Score, 0)
+		t := NewTask(z.Message.Type, z.Message.Payload)
+		tasks = append(tasks, &DeadTask{
+			Task:         t,
+			ID:           z.Message.ID.String(),
+			Queue:        z.Message.Queue,
+			MaxRetry:     z.Message.Retry,
+			Retried:      z.Message.Retried,
+			LastFailedAt: failedAt,
+			ErrorMsg:     z.Message.ErrorMsg,
+			score:        z.Score,
+		})
+	}
+	return tasks, nil
+	return nil, nil
+}
+
+// DeleteAllScheduledTasks deletes all tasks in scheduled state,
+// and reports the number tasks deleted.
+func (i *Inspector) DeleteAllScheduledTasks() (int, error) {
+	n, err := i.rdb.DeleteAllScheduledTasks()
+	return int(n), err
+}
+
+// DeleteAllRetryTasks deletes all tasks in retry state,
+// and reports the number tasks deleted.
+func (i *Inspector) DeleteAllRetryTasks() (int, error) {
+	n, err := i.rdb.DeleteAllRetryTasks()
+	return int(n), err
+}
+
+// DeleteAllDeadTasks deletes all tasks in dead state,
+// and reports the number tasks deleted.
+func (i *Inspector) DeleteAllDeadTasks() (int, error) {
+	n, err := i.rdb.DeleteAllDeadTasks()
+	return int(n), err
+}
+
+// DeleteTaskByKey deletes a task with the given key.
+func (i *Inspector) DeleteTaskByKey(key string) error {
+	id, score, qtype, err := parseTaskKey(key)
+	if err != nil {
+		return err
+	}
+	switch qtype {
+	case "s":
+		return i.rdb.DeleteScheduledTask(id, score)
+	case "r":
+		return i.rdb.DeleteRetryTask(id, score)
+	case "d":
+		return i.rdb.DeleteDeadTask(id, score)
+	default:
+		return fmt.Errorf("invalid key")
+	}
+}
+
+// EnqueueAllScheduledTasks enqueues all tasks in the scheduled state,
+// and reports the number of tasks enqueued.
+func (i *Inspector) EnqueueAllScheduledTasks() (int, error) {
+	n, err := i.rdb.EnqueueAllScheduledTasks()
+	return int(n), err
+}
+
+// EnqueueAllRetryTasks enqueues all tasks in the retry state,
+// and reports the number of tasks enqueued.
+func (i *Inspector) EnqueueAllRetryTasks() (int, error) {
+	n, err := i.rdb.EnqueueAllRetryTasks()
+	return int(n), err
+}
+
+// EnqueueAllDeadTasks enqueues all tasks in the dead state,
+// and reports the number of tasks enqueued.
+func (i *Inspector) EnqueueAllDeadTasks() (int, error) {
+	n, err := i.rdb.EnqueueAllDeadTasks()
+	return int(n), err
+}
+
+// EnqueueTaskByKey enqueues a task with the given key.
+func (i *Inspector) EnqueueTaskByKey(key string) error {
+	id, score, qtype, err := parseTaskKey(key)
+	if err != nil {
+		return err
+	}
+	switch qtype {
+	case "s":
+		return i.rdb.EnqueueScheduledTask(id, score)
+	case "r":
+		return i.rdb.EnqueueRetryTask(id, score)
+	case "d":
+		return i.rdb.EnqueueDeadTask(id, score)
+	default:
+		return fmt.Errorf("invalid key")
+	}
+}
+
+// KillAllScheduledTasks kills all tasks in scheduled state,
+// and reports the number of tasks killed.
+func (i *Inspector) KillAllScheduledTasks() (int, error) {
+	n, err := i.rdb.KillAllScheduledTasks()
+	return int(n), err
+}
+
+// KillAllRetryTasks kills all tasks in retry state,
+// and reports the number of tasks killed.
+func (i *Inspector) KillAllRetryTasks() (int, error) {
+	n, err := i.rdb.KillAllRetryTasks()
+	return int(n), err
+}
+
+// KillTaskByKey kills a task with the given key.
+func (i *Inspector) KillTaskByKey(key string) error {
+	id, score, qtype, err := parseTaskKey(key)
+	if err != nil {
+		return err
+	}
+	switch qtype {
+	case "s":
+		return i.rdb.KillScheduledTask(id, score)
+	case "r":
+		return i.rdb.KillRetryTask(id, score)
+	case "d":
+		return fmt.Errorf("task already dead")
+	default:
+		return fmt.Errorf("invalid key")
+	}
+}
+
+// PauseQueue pauses task processing on the specified queue.
+// If the queue is already paused, it will return a non-nil error.
+func (i *Inspector) PauseQueue(qname string) error {
+	return i.rdb.Pause(qname)
+}
+
+// UnpauseQueue resumes task processing on the specified queue.
+// If the queue is not paused, it will return a non-nil error.
+func (i *Inspector) UnpauseQueue(qname string) error {
+	return i.rdb.Unpause(qname)
+}

internal/asynqtest/asynqtest.go

@@ -13,16 +13,10 @@ import (
 	"github.com/go-redis/redis/v7"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
 	"github.com/hibiken/asynq/internal/base"
-	"github.com/rs/xid"
 )
 
-// ZSetEntry is an entry in redis sorted set.
-type ZSetEntry struct {
-	Msg   *base.TaskMessage
-	Score float64
-}
-
 // SortMsgOpt is a cmp.Option to sort base.TaskMessage for comparing slice of task messages.
 var SortMsgOpt = cmp.Transformer("SortTaskMessages", func(in []*base.TaskMessage) []*base.TaskMessage {
 	out := append([]*base.TaskMessage(nil), in...) // Copy input to avoid mutating it
@@ -33,10 +27,10 @@ var SortMsgOpt = cmp.Transformer("SortTaskMessages", func(in []*base.TaskMessage
 })
 
 // SortZSetEntryOpt is an cmp.Option to sort ZSetEntry for comparing slice of zset entries.
-var SortZSetEntryOpt = cmp.Transformer("SortZSetEntries", func(in []ZSetEntry) []ZSetEntry {
-	out := append([]ZSetEntry(nil), in...) // Copy input to avoid mutating it
+var SortZSetEntryOpt = cmp.Transformer("SortZSetEntries", func(in []base.Z) []base.Z {
+	out := append([]base.Z(nil), in...) // Copy input to avoid mutating it
 	sort.Slice(out, func(i, j int) bool {
-		return out[i].Msg.ID.String() < out[j].Msg.ID.String()
+		return out[i].Message.ID.String() < out[j].Message.ID.String()
 	})
 	return out
 })
@@ -57,7 +51,7 @@ var SortServerInfoOpt = cmp.Transformer("SortServerInfo", func(in []*base.Server
 var SortWorkerInfoOpt = cmp.Transformer("SortWorkerInfo", func(in []*base.WorkerInfo) []*base.WorkerInfo {
 	out := append([]*base.WorkerInfo(nil), in...) // Copy input to avoid mutating it
 	sort.Slice(out, func(i, j int) bool {
-		return out[i].ID.String() < out[j].ID.String()
+		return out[i].ID < out[j].ID
 	})
 	return out
 })
@@ -75,11 +69,13 @@ var IgnoreIDOpt = cmpopts.IgnoreFields(base.TaskMessage{}, "ID")
 // NewTaskMessage returns a new instance of TaskMessage given a task type and payload.
 func NewTaskMessage(taskType string, payload map[string]interface{}) *base.TaskMessage {
 	return &base.TaskMessage{
-		ID:      xid.New(),
-		Type:    taskType,
-		Queue:   base.DefaultQueueName,
-		Retry:   25,
-		Payload: payload,
+		ID:       uuid.New(),
+		Type:     taskType,
+		Queue:    base.DefaultQueueName,
+		Retry:    25,
+		Payload:  payload,
+		Timeout:  1800, // default timeout of 30 mins
+		Deadline: 0,    // no deadline
 	}
 }
 
@@ -87,7 +83,7 @@ func NewTaskMessage(taskType string, payload map[string]interface{}) *base.TaskM
 // task type, payload and queue name.
 func NewTaskMessageWithQueue(taskType string, payload map[string]interface{}, qname string) *base.TaskMessage {
 	return &base.TaskMessage{
-		ID:      xid.New(),
+		ID:      uuid.New(),
 		Type:    taskType,
 		Queue:   qname,
 		Retry:   25,
@@ -95,6 +91,20 @@ func NewTaskMessageWithQueue(taskType string, payload map[string]interface{}, qn
 	}
 }
 
+// TaskMessageAfterRetry returns an updated copy of t after retry.
+// It increments retry count and sets the error message.
+func TaskMessageAfterRetry(t base.TaskMessage, errMsg string) *base.TaskMessage {
+	t.Retried = t.Retried + 1
+	t.ErrorMsg = errMsg
+	return &t
+}
+
+// TaskMessageWithError returns an updated copy of t with the given error message.
+func TaskMessageWithError(t base.TaskMessage, errMsg string) *base.TaskMessage {
+	t.ErrorMsg = errMsg
+	return &t
+}
+
 // MustMarshal marshals given task message and returns a json string.
 // Calling test will fail if marshaling errors out.
 func MustMarshal(tb testing.TB, msg *base.TaskMessage) string {
@@ -161,6 +171,15 @@ func SeedEnqueuedQueue(tb testing.TB, r *redis.Client, msgs []*base.TaskMessage,
 	seedRedisList(tb, r, queue, msgs)
 }
 
+// SeedAllEnqueuedQueues initializes all of the specified queues with the given messages.
+//
+// enqueued maps a queue name to a list of messages.
+func SeedAllEnqueuedQueues(tb testing.TB, r *redis.Client, enqueued map[string][]*base.TaskMessage) {
+	for q, msgs := range enqueued {
+		SeedEnqueuedQueue(tb, r, msgs, q)
+	}
+}
+
 // SeedInProgressQueue initializes the in-progress queue with the given messages.
 func SeedInProgressQueue(tb testing.TB, r *redis.Client, msgs []*base.TaskMessage) {
 	tb.Helper()
@@ -168,23 +187,29 @@ func SeedInProgressQueue(tb testing.TB, r *redis.Client, msgs []*base.TaskMessag
 }
 
 // SeedScheduledQueue initializes the scheduled queue with the given messages.
-func SeedScheduledQueue(tb testing.TB, r *redis.Client, entries []ZSetEntry) {
+func SeedScheduledQueue(tb testing.TB, r *redis.Client, entries []base.Z) {
 	tb.Helper()
 	seedRedisZSet(tb, r, base.ScheduledQueue, entries)
 }
 
 // SeedRetryQueue initializes the retry queue with the given messages.
-func SeedRetryQueue(tb testing.TB, r *redis.Client, entries []ZSetEntry) {
+func SeedRetryQueue(tb testing.TB, r *redis.Client, entries []base.Z) {
 	tb.Helper()
 	seedRedisZSet(tb, r, base.RetryQueue, entries)
 }
 
 // SeedDeadQueue initializes the dead queue with the given messages.
-func SeedDeadQueue(tb testing.TB, r *redis.Client, entries []ZSetEntry) {
+func SeedDeadQueue(tb testing.TB, r *redis.Client, entries []base.Z) {
 	tb.Helper()
 	seedRedisZSet(tb, r, base.DeadQueue, entries)
 }
 
+// SeedDeadlines initializes the deadlines set with the given entries.
+func SeedDeadlines(tb testing.TB, r *redis.Client, entries []base.Z) {
+	tb.Helper()
+	seedRedisZSet(tb, r, base.KeyDeadlines, entries)
+}
+
 func seedRedisList(tb testing.TB, c *redis.Client, key string, msgs []*base.TaskMessage) {
 	data := MustMarshalSlice(tb, msgs)
 	for _, s := range data {
@@ -194,9 +219,9 @@ func seedRedisList(tb testing.TB, c *redis.Client, key string, msgs []*base.Task
 	}
 }
 
-func seedRedisZSet(tb testing.TB, c *redis.Client, key string, items []ZSetEntry) {
+func seedRedisZSet(tb testing.TB, c *redis.Client, key string, items []base.Z) {
 	for _, item := range items {
-		z := &redis.Z{Member: MustMarshal(tb, item.Msg), Score: float64(item.Score)}
+		z := &redis.Z{Member: MustMarshal(tb, item.Message), Score: float64(item.Score)}
 		if err := c.ZAdd(key, z).Err(); err != nil {
 			tb.Fatal(err)
 		}
@@ -240,23 +265,29 @@ func GetDeadMessages(tb testing.TB, r *redis.Client) []*base.TaskMessage {
 }
 
 // GetScheduledEntries returns all task messages and its score in the scheduled queue.
-func GetScheduledEntries(tb testing.TB, r *redis.Client) []ZSetEntry {
+func GetScheduledEntries(tb testing.TB, r *redis.Client) []base.Z {
 	tb.Helper()
 	return getZSetEntries(tb, r, base.ScheduledQueue)
 }
 
 // GetRetryEntries returns all task messages and its score in the retry queue.
-func GetRetryEntries(tb testing.TB, r *redis.Client) []ZSetEntry {
+func GetRetryEntries(tb testing.TB, r *redis.Client) []base.Z {
 	tb.Helper()
 	return getZSetEntries(tb, r, base.RetryQueue)
 }
 
 // GetDeadEntries returns all task messages and its score in the dead queue.
-func GetDeadEntries(tb testing.TB, r *redis.Client) []ZSetEntry {
+func GetDeadEntries(tb testing.TB, r *redis.Client) []base.Z {
 	tb.Helper()
 	return getZSetEntries(tb, r, base.DeadQueue)
 }
 
+// GetDeadlinesEntries returns all task messages and its score in the deadlines set.
+func GetDeadlinesEntries(tb testing.TB, r *redis.Client) []base.Z {
+	tb.Helper()
+	return getZSetEntries(tb, r, base.KeyDeadlines)
+}
+
 func getListMessages(tb testing.TB, r *redis.Client, list string) []*base.TaskMessage {
 	data := r.LRange(list, 0, -1).Val()
 	return MustUnmarshalSlice(tb, data)
@@ -267,13 +298,13 @@ func getZSetMessages(tb testing.TB, r *redis.Client, zset string) []*base.TaskMe
 	return MustUnmarshalSlice(tb, data)
 }
 
-func getZSetEntries(tb testing.TB, r *redis.Client, zset string) []ZSetEntry {
+func getZSetEntries(tb testing.TB, r *redis.Client, zset string) []base.Z {
 	data := r.ZRangeWithScores(zset, 0, -1).Val()
-	var entries []ZSetEntry
+	var entries []base.Z
 	for _, z := range data {
-		entries = append(entries, ZSetEntry{
-			Msg:   MustUnmarshal(tb, z.Member.(string)),
-			Score: z.Score,
+		entries = append(entries, base.Z{
+			Message: MustUnmarshal(tb, z.Member.(string)),
+			Score:   int64(z.Score),
 		})
 	}
 	return entries

internal/base/base.go

@@ -7,15 +7,19 @@ package base
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/go-redis/redis/v7"
-	"github.com/rs/xid"
+	"github.com/google/uuid"
 )
 
+// Version of asynq library and CLI.
+const Version = "0.10.0"
+
 // DefaultQueueName is the queue name used if none are specified by user.
 const DefaultQueueName = "default"
 
@@ -34,6 +38,8 @@ const (
 	RetryQueue      = "asynq:retry"                  // ZSET
 	DeadQueue       = "asynq:dead"                   // ZSET
 	InProgressQueue = "asynq:in_progress"            // LIST
+	KeyDeadlines    = "asynq:deadlines"              // ZSET
+	PausedQueues    = "asynq:paused"                 // SET
 	CancelChannel   = "asynq:cancel"                 // PubSub channel
 )
 
@@ -72,7 +78,7 @@ type TaskMessage struct {
 	Payload map[string]interface{}
 
 	// ID is a unique identifier for each task.
-	ID xid.ID
+	ID uuid.UUID
 
 	// Queue is a name this message should be enqueued to.
 	Queue string
@@ -86,18 +92,20 @@ type TaskMessage struct {
 	// ErrorMsg holds the error message from the last failure.
 	ErrorMsg string
 
-	// Timeout specifies how long a task may run.
-	// The string value should be compatible with time.Duration.ParseDuration.
+	// Timeout specifies timeout in seconds.
+	// If task processing doesn't complete within the timeout, the task will be retried
+	// if retry count is remaining. Otherwise it will be moved to the dead queue.
 	//
-	// Zero means no limit.
-	Timeout string
+	// Use zero to indicate no timeout.
+	Timeout int64
 
-	// Deadline specifies the deadline for the task.
-	// Task won't be processed if it exceeded its deadline.
-	// The string shoulbe be in RFC3339 format.
+	// Deadline specifies the deadline for the task in Unix time,
+	// the number of seconds elapsed since January 1, 1970 UTC.
+	// If task processing doesn't complete before the deadline, the task will be retried
+	// if retry count is remaining. Otherwise it will be moved to the dead queue.
 	//
-	// time.Time's zero value means no deadline.
-	Deadline string
+	// Use zero to indicate no deadline.
+	Deadline int64
 
 	// UniqueKey holds the redis key used for uniqueness lock for this task.
 	//
@@ -105,28 +113,49 @@ type TaskMessage struct {
 	UniqueKey string
 }
 
-// ServerState holds process level information.
-//
-// ServerStates are safe for concurrent use by multiple goroutines.
-type ServerState struct {
-	mu             sync.Mutex // guards all data fields
-	id             xid.ID
-	concurrency    int
-	queues         map[string]int
-	strictPriority bool
-	pid            int
-	host           string
-	status         ServerStatus
-	started        time.Time
-	workers        map[string]*workerStats
+// EncodeMessage marshals the given task message in JSON and returns an encoded string.
+func EncodeMessage(msg *TaskMessage) (string, error) {
+	b, err := json.Marshal(msg)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}
+
+// DecodeMessage unmarshals the given encoded string and returns a decoded task message.
+func DecodeMessage(s string) (*TaskMessage, error) {
+	d := json.NewDecoder(strings.NewReader(s))
+	d.UseNumber()
+	var msg TaskMessage
+	if err := d.Decode(&msg); err != nil {
+		return nil, err
+	}
+	return &msg, nil
+}
+
+// Z represents sorted set member.
+type Z struct {
+	Message *TaskMessage
+	Score   int64
 }
 
 // ServerStatus represents status of a server.
-type ServerStatus int
+// ServerStatus methods are concurrency safe.
+type ServerStatus struct {
+	mu  sync.Mutex
+	val ServerStatusValue
+}
+
+// NewServerStatus returns a new status instance given an initial value.
+func NewServerStatus(v ServerStatusValue) *ServerStatus {
+	return &ServerStatus{val: v}
+}
+
+type ServerStatusValue int
 
 const (
 	// StatusIdle indicates the server is in idle state.
-	StatusIdle ServerStatus = iota
+	StatusIdle ServerStatusValue = iota
 
 	// StatusRunning indicates the servier is up and processing tasks.
 	StatusRunning
@@ -145,117 +174,28 @@ var statuses = []string{
 	"stopped",
 }
 
-func (s ServerStatus) String() string {
-	if StatusIdle <= s && s <= StatusStopped {
-		return statuses[s]
+func (s *ServerStatus) String() string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if StatusIdle <= s.val && s.val <= StatusStopped {
+		return statuses[s.val]
 	}
 	return "unknown status"
 }
 
-type workerStats struct {
-	msg     *TaskMessage
-	started time.Time
+// Get returns the status value.
+func (s *ServerStatus) Get() ServerStatusValue {
+	s.mu.Lock()
+	v := s.val
+	s.mu.Unlock()
+	return v
 }
 
-// NewServerState returns a new instance of ServerState.
-func NewServerState(host string, pid, concurrency int, queues map[string]int, strict bool) *ServerState {
-	return &ServerState{
-		host:           host,
-		pid:            pid,
-		id:             xid.New(),
-		concurrency:    concurrency,
-		queues:         cloneQueueConfig(queues),
-		strictPriority: strict,
-		status:         StatusIdle,
-		workers:        make(map[string]*workerStats),
-	}
-}
-
-// SetStatus updates the status of server.
-func (ss *ServerState) SetStatus(status ServerStatus) {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	ss.status = status
-}
-
-// Status returns the status of server.
-func (ss *ServerState) Status() ServerStatus {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	return ss.status
-}
-
-// SetStarted records when the process started processing.
-func (ss *ServerState) SetStarted(t time.Time) {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	ss.started = t
-}
-
-// AddWorkerStats records when a worker started and which task it's processing.
-func (ss *ServerState) AddWorkerStats(msg *TaskMessage, started time.Time) {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	ss.workers[msg.ID.String()] = &workerStats{msg, started}
-}
-
-// DeleteWorkerStats removes a worker's entry from the process state.
-func (ss *ServerState) DeleteWorkerStats(msg *TaskMessage) {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	delete(ss.workers, msg.ID.String())
-}
-
-// GetInfo returns current state of server as a ServerInfo.
-func (ss *ServerState) GetInfo() *ServerInfo {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	return &ServerInfo{
-		Host:              ss.host,
-		PID:               ss.pid,
-		ServerID:          ss.id.String(),
-		Concurrency:       ss.concurrency,
-		Queues:            cloneQueueConfig(ss.queues),
-		StrictPriority:    ss.strictPriority,
-		Status:            ss.status.String(),
-		Started:           ss.started,
-		ActiveWorkerCount: len(ss.workers),
-	}
-}
-
-// GetWorkers returns a list of currently running workers' info.
-func (ss *ServerState) GetWorkers() []*WorkerInfo {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	var res []*WorkerInfo
-	for _, w := range ss.workers {
-		res = append(res, &WorkerInfo{
-			Host:    ss.host,
-			PID:     ss.pid,
-			ID:      w.msg.ID,
-			Type:    w.msg.Type,
-			Queue:   w.msg.Queue,
-			Payload: clonePayload(w.msg.Payload),
-			Started: w.started,
-		})
-	}
-	return res
-}
-
-func cloneQueueConfig(qcfg map[string]int) map[string]int {
-	res := make(map[string]int)
-	for qname, n := range qcfg {
-		res[qname] = n
-	}
-	return res
-}
-
-func clonePayload(payload map[string]interface{}) map[string]interface{} {
-	res := make(map[string]interface{})
-	for k, v := range payload {
-		res[k] = v
-	}
-	return res
+// Set sets the status value.
+func (s *ServerStatus) Set(v ServerStatusValue) {
+	s.mu.Lock()
+	s.val = v
+	s.mu.Unlock()
 }
 
 // ServerInfo holds information about a running server.
@@ -275,7 +215,7 @@ type ServerInfo struct {
 type WorkerInfo struct {
 	Host    string
 	PID     int
-	ID      xid.ID
+	ID      string
 	Type    string
 	Queue   string
 	Payload map[string]interface{}
@@ -319,34 +259,24 @@ func (c *Cancelations) Get(id string) (fn context.CancelFunc, ok bool) {
 	return fn, ok
 }
 
-// GetAll returns all cancel funcs.
-func (c *Cancelations) GetAll() []context.CancelFunc {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	var res []context.CancelFunc
-	for _, fn := range c.cancelFuncs {
-		res = append(res, fn)
-	}
-	return res
-}
-
 // Broker is a message broker that supports operations to manage task queues.
 //
 // See rdb.RDB as a reference implementation.
 type Broker interface {
+	Ping() error
 	Enqueue(msg *TaskMessage) error
 	EnqueueUnique(msg *TaskMessage, ttl time.Duration) error
-	Dequeue(qnames ...string) (*TaskMessage, error)
+	Dequeue(qnames ...string) (*TaskMessage, time.Time, error)
 	Done(msg *TaskMessage) error
 	Requeue(msg *TaskMessage) error
 	Schedule(msg *TaskMessage, processAt time.Time) error
 	ScheduleUnique(msg *TaskMessage, processAt time.Time, ttl time.Duration) error
 	Retry(msg *TaskMessage, processAt time.Time, errMsg string) error
 	Kill(msg *TaskMessage, errMsg string) error
-	RequeueAll() (int64, error)
-	CheckAndEnqueue(qnames ...string) error
-	WriteServerState(ss *ServerState, ttl time.Duration) error
-	ClearServerState(ss *ServerState) error
+	CheckAndEnqueue() error
+	ListDeadlineExceeded(deadline time.Time) ([]*TaskMessage, error)
+	WriteServerState(info *ServerInfo, workers []*WorkerInfo, ttl time.Duration) error
+	ClearServerState(host string, pid int, serverID string) error
 	CancelationPubSub() (*redis.PubSub, error) // TODO: Need to decouple from redis to support other brokers
 	PublishCancelation(id string) error
 	Close() error

internal/base/base_test.go

@@ -6,14 +6,13 @@ package base
 
 import (
 	"context"
-	"math/rand"
+	"encoding/json"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
-	"github.com/rs/xid"
+	"github.com/google/uuid"
 )
 
 func TestQueueKey(t *testing.T) {
@@ -108,69 +107,76 @@ func TestWorkersKey(t *testing.T) {
 	}
 }
 
-// Test for server state being accessed by multiple goroutines.
-// Run with -race flag to check for data race.
-func TestServerStateConcurrentAccess(t *testing.T) {
-	ss := NewServerState("127.0.0.1", 1234, 10, map[string]int{"default": 1}, false)
-	var wg sync.WaitGroup
-	started := time.Now()
-	msgs := []*TaskMessage{
-		{ID: xid.New(), Type: "type1", Payload: map[string]interface{}{"user_id": 42}},
-		{ID: xid.New(), Type: "type2"},
-		{ID: xid.New(), Type: "type3"},
+func TestMessageEncoding(t *testing.T) {
+	id := uuid.New()
+	tests := []struct {
+		in  *TaskMessage
+		out *TaskMessage
+	}{
+		{
+			in: &TaskMessage{
+				Type:     "task1",
+				Payload:  map[string]interface{}{"a": 1, "b": "hello!", "c": true},
+				ID:       id,
+				Queue:    "default",
+				Retry:    10,
+				Retried:  0,
+				Timeout:  1800,
+				Deadline: 1692311100,
+			},
+			out: &TaskMessage{
+				Type:     "task1",
+				Payload:  map[string]interface{}{"a": json.Number("1"), "b": "hello!", "c": true},
+				ID:       id,
+				Queue:    "default",
+				Retry:    10,
+				Retried:  0,
+				Timeout:  1800,
+				Deadline: 1692311100,
+			},
+		},
 	}
 
-	// Simulate hearbeater calling SetStatus and SetStarted.
+	for _, tc := range tests {
+		encoded, err := EncodeMessage(tc.in)
+		if err != nil {
+			t.Errorf("EncodeMessage(msg) returned error: %v", err)
+			continue
+		}
+		decoded, err := DecodeMessage(encoded)
+		if err != nil {
+			t.Errorf("DecodeMessage(encoded) returned error: %v", err)
+			continue
+		}
+		if diff := cmp.Diff(tc.out, decoded); diff != "" {
+			t.Errorf("Decoded message == %+v, want %+v;(-want,+got)\n%s",
+				decoded, tc.out, diff)
+		}
+	}
+}
+
+// Test for status being accessed by multiple goroutines.
+// Run with -race flag to check for data race.
+func TestStatusConcurrentAccess(t *testing.T) {
+	status := NewServerStatus(StatusIdle)
+
+	var wg sync.WaitGroup
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		ss.SetStarted(started)
-		ss.SetStatus(StatusRunning)
-		if status := ss.Status(); status != StatusRunning {
-			t.Errorf("(*ServerState).Status() = %v, want %v", status, StatusRunning)
-		}
+		status.Get()
+		status.String()
 	}()
 
-	// Simulate processor starting worker goroutines.
-	for _, msg := range msgs {
-		wg.Add(1)
-		ss.AddWorkerStats(msg, time.Now())
-		go func(msg *TaskMessage) {
-			defer wg.Done()
-			time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond)
-			ss.DeleteWorkerStats(msg)
-		}(msg)
-	}
-
-	// Simulate hearbeater calling Get and GetWorkers
 	wg.Add(1)
 	go func() {
-		wg.Done()
-		for i := 0; i < 5; i++ {
-			ss.GetInfo()
-			ss.GetWorkers()
-			time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
-		}
+		defer wg.Done()
+		status.Set(StatusStopped)
+		status.String()
 	}()
 
 	wg.Wait()
-
-	want := &ServerInfo{
-		Host:              "127.0.0.1",
-		PID:               1234,
-		Concurrency:       10,
-		Queues:            map[string]int{"default": 1},
-		StrictPriority:    false,
-		Status:            "running",
-		Started:           started,
-		ActiveWorkerCount: 0,
-	}
-
-	got := ss.GetInfo()
-	if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(ServerInfo{}, "ServerID")); diff != "" {
-		t.Errorf("(*ServerState).GetInfo() = %+v, want %+v; (-want,+got)\n%s",
-			got, want, diff)
-	}
 }
 
 // Test for cancelations being accessed by multiple goroutines.
@@ -216,9 +222,4 @@ func TestCancelationsConcurrentAccess(t *testing.T) {
 	if ok {
 		t.Errorf("(*Cancelations).Get(%q) = _, true, want <nil>, false", key2)
 	}
-
-	funcs := c.GetAll()
-	if len(funcs) != 2 {
-		t.Errorf("(*Cancelations).GetAll() returns %d functions, want 2", len(funcs))
-	}
 }

internal/rdb/inspect.go

@@ -7,12 +7,13 @@ package rdb
 import (
 	"encoding/json"
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 
 	"github.com/go-redis/redis/v7"
+	"github.com/google/uuid"
 	"github.com/hibiken/asynq/internal/base"
-	"github.com/rs/xid"
 	"github.com/spf13/cast"
 )
 
@@ -25,10 +26,24 @@ type Stats struct {
 	Dead       int
 	Processed  int
 	Failed     int
-	Queues     map[string]int // map of queue name to number of tasks in the queue (e.g., "default": 100, "critical": 20)
+	Queues     []*Queue
 	Timestamp  time.Time
 }
 
+// Queue represents a task queue.
+type Queue struct {
+	// Name of the queue (e.g. "default", "critical").
+	// Note: It doesn't include the prefix "asynq:queues:".
+	Name string
+
+	// Paused indicates whether the queue is paused.
+	// If true, tasks in the queue should not be processed.
+	Paused bool
+
+	// Size is the number of tasks in the queue.
+	Size int
+}
+
 // DailyStats holds aggregate data for a given day.
 type DailyStats struct {
 	Processed int
@@ -36,56 +51,6 @@ type DailyStats struct {
 	Time      time.Time
 }
 
-// EnqueuedTask is a task in a queue and is ready to be processed.
-type EnqueuedTask struct {
-	ID      xid.ID
-	Type    string
-	Payload map[string]interface{}
-	Queue   string
-}
-
-// InProgressTask is a task that's currently being processed.
-type InProgressTask struct {
-	ID      xid.ID
-	Type    string
-	Payload map[string]interface{}
-}
-
-// ScheduledTask is a task that's scheduled to be processed in the future.
-type ScheduledTask struct {
-	ID        xid.ID
-	Type      string
-	Payload   map[string]interface{}
-	ProcessAt time.Time
-	Score     int64
-	Queue     string
-}
-
-// RetryTask is a task that's in retry queue because worker failed to process the task.
-type RetryTask struct {
-	ID      xid.ID
-	Type    string
-	Payload map[string]interface{}
-	// TODO(hibiken): add LastFailedAt time.Time
-	ProcessAt time.Time
-	ErrorMsg  string
-	Retried   int
-	Retry     int
-	Score     int64
-	Queue     string
-}
-
-// DeadTask is a task in that has exhausted all retries.
-type DeadTask struct {
-	ID           xid.ID
-	Type         string
-	Payload      map[string]interface{}
-	LastFailedAt time.Time
-	ErrorMsg     string
-	Score        int64
-	Queue        string
-}
-
 // KEYS[1] -> asynq:queues
 // KEYS[2] -> asynq:in_progress
 // KEYS[3] -> asynq:scheduled
@@ -143,8 +108,12 @@ func (r *RDB) CurrentStats() (*Stats, error) {
 	if err != nil {
 		return nil, err
 	}
+	paused, err := r.client.SMembersMap(base.PausedQueues).Result()
+	if err != nil {
+		return nil, err
+	}
 	stats := &Stats{
-		Queues:    make(map[string]int),
+		Queues:    make([]*Queue, 0),
 		Timestamp: now,
 	}
 	for i := 0; i < len(data); i += 2 {
@@ -154,7 +123,14 @@ func (r *RDB) CurrentStats() (*Stats, error) {
 		switch {
 		case strings.HasPrefix(key, base.QueuePrefix):
 			stats.Enqueued += val
-			stats.Queues[strings.TrimPrefix(key, base.QueuePrefix)] = val
+			q := Queue{
+				Name: strings.TrimPrefix(key, base.QueuePrefix),
+				Size: val,
+			}
+			if _, exist := paused[key]; exist {
+				q.Paused = true
+			}
+			stats.Queues = append(stats.Queues, &q)
 		case key == base.InProgressQueue:
 			stats.InProgress = val
 		case key == base.ScheduledQueue:
@@ -169,6 +145,9 @@ func (r *RDB) CurrentStats() (*Stats, error) {
 			stats.Failed = val
 		}
 	}
+	sort.Slice(stats.Queues, func(i, j int) bool {
+		return stats.Queues[i].Name < stats.Queues[j].Name
+	})
 	return stats, nil
 }
 
@@ -260,164 +239,85 @@ func (p Pagination) stop() int64 {
 }
 
 // ListEnqueued returns enqueued tasks that are ready to be processed.
-func (r *RDB) ListEnqueued(qname string, pgn Pagination) ([]*EnqueuedTask, error) {
+func (r *RDB) ListEnqueued(qname string, pgn Pagination) ([]*base.TaskMessage, error) {
 	qkey := base.QueueKey(qname)
 	if !r.client.SIsMember(base.AllQueues, qkey).Val() {
 		return nil, fmt.Errorf("queue %q does not exist", qname)
 	}
-	// Note: Because we use LPUSH to redis list, we need to calculate the
-	// correct range and reverse the list to get the tasks with pagination.
-	stop := -pgn.start() - 1
-	start := -pgn.stop() - 1
-	data, err := r.client.LRange(qkey, start, stop).Result()
-	if err != nil {
-		return nil, err
-	}
-	reverse(data)
-	var tasks []*EnqueuedTask
-	for _, s := range data {
-		var msg base.TaskMessage
-		err := json.Unmarshal([]byte(s), &msg)
-		if err != nil {
-			continue // bad data, ignore and continue
-		}
-		tasks = append(tasks, &EnqueuedTask{
-			ID:      msg.ID,
-			Type:    msg.Type,
-			Payload: msg.Payload,
-			Queue:   msg.Queue,
-		})
-	}
-	return tasks, nil
+	return r.listMessages(qkey, pgn)
 }
 
 // ListInProgress returns all tasks that are currently being processed.
-func (r *RDB) ListInProgress(pgn Pagination) ([]*InProgressTask, error) {
+func (r *RDB) ListInProgress(pgn Pagination) ([]*base.TaskMessage, error) {
+	return r.listMessages(base.InProgressQueue, pgn)
+}
+
+// listMessages returns a list of TaskMessage in Redis list with the given key.
+func (r *RDB) listMessages(key string, pgn Pagination) ([]*base.TaskMessage, error) {
 	// Note: Because we use LPUSH to redis list, we need to calculate the
 	// correct range and reverse the list to get the tasks with pagination.
 	stop := -pgn.start() - 1
 	start := -pgn.stop() - 1
-	data, err := r.client.LRange(base.InProgressQueue, start, stop).Result()
+	data, err := r.client.LRange(key, start, stop).Result()
 	if err != nil {
 		return nil, err
 	}
 	reverse(data)
-	var tasks []*InProgressTask
+	var msgs []*base.TaskMessage
 	for _, s := range data {
-		var msg base.TaskMessage
-		err := json.Unmarshal([]byte(s), &msg)
+		m, err := base.DecodeMessage(s)
 		if err != nil {
 			continue // bad data, ignore and continue
 		}
-		tasks = append(tasks, &InProgressTask{
-			ID:      msg.ID,
-			Type:    msg.Type,
-			Payload: msg.Payload,
-		})
+		msgs = append(msgs, m)
 	}
-	return tasks, nil
+	return msgs, nil
+
 }
 
 // ListScheduled returns all tasks that are scheduled to be processed
 // in the future.
-func (r *RDB) ListScheduled(pgn Pagination) ([]*ScheduledTask, error) {
-	data, err := r.client.ZRangeWithScores(base.ScheduledQueue, pgn.start(), pgn.stop()).Result()
-	if err != nil {
-		return nil, err
-	}
-	var tasks []*ScheduledTask
-	for _, z := range data {
-		s, ok := z.Member.(string)
-		if !ok {
-			continue // bad data, ignore and continue
-		}
-		var msg base.TaskMessage
-		err := json.Unmarshal([]byte(s), &msg)
-		if err != nil {
-			continue // bad data, ignore and continue
-		}
-		processAt := time.Unix(int64(z.Score), 0)
-		tasks = append(tasks, &ScheduledTask{
-			ID:        msg.ID,
-			Type:      msg.Type,
-			Payload:   msg.Payload,
-			Queue:     msg.Queue,
-			ProcessAt: processAt,
-			Score:     int64(z.Score),
-		})
-	}
-	return tasks, nil
+func (r *RDB) ListScheduled(pgn Pagination) ([]base.Z, error) {
+	return r.listZSetEntries(base.ScheduledQueue, pgn)
 }
 
 // ListRetry returns all tasks that have failed before and willl be retried
 // in the future.
-func (r *RDB) ListRetry(pgn Pagination) ([]*RetryTask, error) {
-	data, err := r.client.ZRangeWithScores(base.RetryQueue, pgn.start(), pgn.stop()).Result()
-	if err != nil {
-		return nil, err
-	}
-	var tasks []*RetryTask
-	for _, z := range data {
-		s, ok := z.Member.(string)
-		if !ok {
-			continue // bad data, ignore and continue
-		}
-		var msg base.TaskMessage
-		err := json.Unmarshal([]byte(s), &msg)
-		if err != nil {
-			continue // bad data, ignore and continue
-		}
-		processAt := time.Unix(int64(z.Score), 0)
-		tasks = append(tasks, &RetryTask{
-			ID:        msg.ID,
-			Type:      msg.Type,
-			Payload:   msg.Payload,
-			ErrorMsg:  msg.ErrorMsg,
-			Retry:     msg.Retry,
-			Retried:   msg.Retried,
-			Queue:     msg.Queue,
-			ProcessAt: processAt,
-			Score:     int64(z.Score),
-		})
-	}
-	return tasks, nil
+func (r *RDB) ListRetry(pgn Pagination) ([]base.Z, error) {
+	return r.listZSetEntries(base.RetryQueue, pgn)
 }
 
 // ListDead returns all tasks that have exhausted its retry limit.
-func (r *RDB) ListDead(pgn Pagination) ([]*DeadTask, error) {
-	data, err := r.client.ZRangeWithScores(base.DeadQueue, pgn.start(), pgn.stop()).Result()
+func (r *RDB) ListDead(pgn Pagination) ([]base.Z, error) {
+	return r.listZSetEntries(base.DeadQueue, pgn)
+}
+
+// listZSetEntries returns a list of message and score pairs in Redis sorted-set
+// with the given key.
+func (r *RDB) listZSetEntries(key string, pgn Pagination) ([]base.Z, error) {
+	data, err := r.client.ZRangeWithScores(key, pgn.start(), pgn.stop()).Result()
 	if err != nil {
 		return nil, err
 	}
-	var tasks []*DeadTask
+	var res []base.Z
 	for _, z := range data {
 		s, ok := z.Member.(string)
 		if !ok {
 			continue // bad data, ignore and continue
 		}
-		var msg base.TaskMessage
-		err := json.Unmarshal([]byte(s), &msg)
+		msg, err := base.DecodeMessage(s)
 		if err != nil {
 			continue // bad data, ignore and continue
 		}
-		lastFailedAt := time.Unix(int64(z.Score), 0)
-		tasks = append(tasks, &DeadTask{
-			ID:           msg.ID,
-			Type:         msg.Type,
-			Payload:      msg.Payload,
-			ErrorMsg:     msg.ErrorMsg,
-			Queue:        msg.Queue,
-			LastFailedAt: lastFailedAt,
-			Score:        int64(z.Score),
-		})
+		res = append(res, base.Z{msg, int64(z.Score)})
 	}
-	return tasks, nil
+	return res, nil
 }
 
 // EnqueueDeadTask finds a task that matches the given id and score from dead queue
 // and enqueues it for processing. If a task that matches the id and score
 // does not exist, it returns ErrTaskNotFound.
-func (r *RDB) EnqueueDeadTask(id xid.ID, score int64) error {
+func (r *RDB) EnqueueDeadTask(id uuid.UUID, score int64) error {
 	n, err := r.removeAndEnqueue(base.DeadQueue, id.String(), float64(score))
 	if err != nil {
 		return err
@@ -431,7 +331,7 @@ func (r *RDB) EnqueueDeadTask(id xid.ID, score int64) error {
 // EnqueueRetryTask finds a task that matches the given id and score from retry queue
 // and enqueues it for processing. If a task that matches the id and score
 // does not exist, it returns ErrTaskNotFound.
-func (r *RDB) EnqueueRetryTask(id xid.ID, score int64) error {
+func (r *RDB) EnqueueRetryTask(id uuid.UUID, score int64) error {
 	n, err := r.removeAndEnqueue(base.RetryQueue, id.String(), float64(score))
 	if err != nil {
 		return err
@@ -445,7 +345,7 @@ func (r *RDB) EnqueueRetryTask(id xid.ID, score int64) error {
 // EnqueueScheduledTask finds a task that matches the given id and score from scheduled queue
 // and enqueues it for processing. If a task that matches the id and score does not
 // exist, it returns ErrTaskNotFound.
-func (r *RDB) EnqueueScheduledTask(id xid.ID, score int64) error {
+func (r *RDB) EnqueueScheduledTask(id uuid.UUID, score int64) error {
 	n, err := r.removeAndEnqueue(base.ScheduledQueue, id.String(), float64(score))
 	if err != nil {
 		return err
@@ -524,7 +424,7 @@ func (r *RDB) removeAndEnqueueAll(zset string) (int64, error) {
 // KillRetryTask finds a task that matches the given id and score from retry queue
 // and moves it to dead queue. If a task that maches the id and score does not exist,
 // it returns ErrTaskNotFound.
-func (r *RDB) KillRetryTask(id xid.ID, score int64) error {
+func (r *RDB) KillRetryTask(id uuid.UUID, score int64) error {
 	n, err := r.removeAndKill(base.RetryQueue, id.String(), float64(score))
 	if err != nil {
 		return err
@@ -538,7 +438,7 @@ func (r *RDB) KillRetryTask(id xid.ID, score int64) error {
 // KillScheduledTask finds a task that matches the given id and score from scheduled queue
 // and moves it to dead queue. If a task that maches the id and score does not exist,
 // it returns ErrTaskNotFound.
-func (r *RDB) KillScheduledTask(id xid.ID, score int64) error {
+func (r *RDB) KillScheduledTask(id uuid.UUID, score int64) error {
 	n, err := r.removeAndKill(base.ScheduledQueue, id.String(), float64(score))
 	if err != nil {
 		return err
@@ -631,21 +531,21 @@ func (r *RDB) removeAndKillAll(zset string) (int64, error) {
 // DeleteDeadTask finds a task that matches the given id and score from dead queue
 // and deletes it. If a task that matches the id and score does not exist,
 // it returns ErrTaskNotFound.
-func (r *RDB) DeleteDeadTask(id xid.ID, score int64) error {
+func (r *RDB) DeleteDeadTask(id uuid.UUID, score int64) error {
 	return r.deleteTask(base.DeadQueue, id.String(), float64(score))
 }
 
 // DeleteRetryTask finds a task that matches the given id and score from retry queue
 // and deletes it. If a task that matches the id and score does not exist,
 // it returns ErrTaskNotFound.
-func (r *RDB) DeleteRetryTask(id xid.ID, score int64) error {
+func (r *RDB) DeleteRetryTask(id uuid.UUID, score int64) error {
 	return r.deleteTask(base.RetryQueue, id.String(), float64(score))
 }
 
 // DeleteScheduledTask finds a task that matches the given id and score from
 // scheduled queue  and deletes it. If a task that matches the id and score
 //does not exist, it returns ErrTaskNotFound.
-func (r *RDB) DeleteScheduledTask(id xid.ID, score int64) error {
+func (r *RDB) DeleteScheduledTask(id uuid.UUID, score int64) error {
 	return r.deleteTask(base.ScheduledQueue, id.String(), float64(score))
 }
 
@@ -675,19 +575,40 @@ func (r *RDB) deleteTask(zset, id string, score float64) error {
 	return nil
 }
 
-// DeleteAllDeadTasks deletes all tasks from the dead queue.
-func (r *RDB) DeleteAllDeadTasks() error {
-	return r.client.Del(base.DeadQueue).Err()
+// KEYS[1] -> queue to delete
+var deleteAllCmd = redis.NewScript(`
+local n = redis.call("ZCARD", KEYS[1])
+redis.call("DEL", KEYS[1])
+return n`)
+
+// DeleteAllDeadTasks deletes all tasks from the dead queue
+// and returns the number of tasks deleted.
+func (r *RDB) DeleteAllDeadTasks() (int64, error) {
+	return r.deleteAll(base.DeadQueue)
 }
 
-// DeleteAllRetryTasks deletes all tasks from the dead queue.
-func (r *RDB) DeleteAllRetryTasks() error {
-	return r.client.Del(base.RetryQueue).Err()
+// DeleteAllRetryTasks deletes all tasks from the dead queue
+// and returns the number of tasks deleted.
+func (r *RDB) DeleteAllRetryTasks() (int64, error) {
+	return r.deleteAll(base.RetryQueue)
 }
 
-// DeleteAllScheduledTasks deletes all tasks from the dead queue.
-func (r *RDB) DeleteAllScheduledTasks() error {
-	return r.client.Del(base.ScheduledQueue).Err()
+// DeleteAllScheduledTasks deletes all tasks from the dead queue
+// and returns the number of tasks deleted.
+func (r *RDB) DeleteAllScheduledTasks() (int64, error) {
+	return r.deleteAll(base.ScheduledQueue)
+}
+
+func (r *RDB) deleteAll(key string) (int64, error) {
+	res, err := deleteAllCmd.Run(r.client, []string{key}).Result()
+	if err != nil {
+		return 0, err
+	}
+	n, ok := res.(int64)
+	if !ok {
+		return 0, fmt.Errorf("could not cast %v to int64", res)
+	}
+	return n, nil
 }
 
 // ErrQueueNotFound indicates specified queue does not exist.
@@ -830,3 +751,33 @@ func (r *RDB) ListWorkers() ([]*base.WorkerInfo, error) {
 	}
 	return workers, nil
 }
+
+// KEYS[1] -> asynq:paused
+// ARGV[1] -> asynq:queues:<qname> - queue to pause
+var pauseCmd = redis.NewScript(`
+local ismem = redis.call("SISMEMBER", KEYS[1], ARGV[1])
+if ismem == 1 then
+	return redis.error_reply("queue is already paused")
+end
+return redis.call("SADD", KEYS[1], ARGV[1])`)
+
+// Pause pauses processing of tasks from the given queue.
+func (r *RDB) Pause(qname string) error {
+	qkey := base.QueueKey(qname)
+	return pauseCmd.Run(r.client, []string{base.PausedQueues}, qkey).Err()
+}
+
+// KEYS[1] -> asynq:paused
+// ARGV[1] -> asynq:queues:<qname> - queue to unpause
+var unpauseCmd = redis.NewScript(`
+local ismem = redis.call("SISMEMBER", KEYS[1], ARGV[1])
+if ismem == 0 then
+	return redis.error_reply("queue is not paused")
+end
+return redis.call("SREM", KEYS[1], ARGV[1])`)
+
+// Unpause resumes processing of tasks from the given queue.
+func (r *RDB) Unpause(qname string) error {
+	qkey := base.QueueKey(qname)
+	return unpauseCmd.Run(r.client, []string{base.PausedQueues}, qkey).Err()
+}

internal/rdb/rdb.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"strconv"
 	"time"
 
 	"github.com/go-redis/redis/v7"
@@ -44,6 +45,11 @@ func (r *RDB) Close() error {
 	return r.client.Close()
 }
 
+// Ping checks the connection with redis server.
+func (r *RDB) Ping() error {
+	return r.client.Ping().Err()
+}
+
 // KEYS[1] -> asynq:queues:<qname>
 // KEYS[2] -> asynq:queues
 // ARGV[1] -> task message data
@@ -54,12 +60,12 @@ return 1`)
 
 // Enqueue inserts the given task to the tail of the queue.
 func (r *RDB) Enqueue(msg *base.TaskMessage) error {
-	bytes, err := json.Marshal(msg)
+	encoded, err := base.EncodeMessage(msg)
 	if err != nil {
 		return err
 	}
 	key := base.QueueKey(msg.Queue)
-	return enqueueCmd.Run(r.client, []string{key, base.AllQueues}, bytes).Err()
+	return enqueueCmd.Run(r.client, []string{key, base.AllQueues}, encoded).Err()
 }
 
 // KEYS[1] -> unique key in the form <type>:<payload>:<qname>
@@ -81,14 +87,14 @@ return 1
 // EnqueueUnique inserts the given task if the task's uniqueness lock can be acquired.
 // It returns ErrDuplicateTask if the lock cannot be acquired.
 func (r *RDB) EnqueueUnique(msg *base.TaskMessage, ttl time.Duration) error {
-	bytes, err := json.Marshal(msg)
+	encoded, err := base.EncodeMessage(msg)
 	if err != nil {
 		return err
 	}
 	key := base.QueueKey(msg.Queue)
 	res, err := enqueueUniqueCmd.Run(r.client,
 		[]string{msg.UniqueKey, key, base.AllQueues},
-		msg.ID.String(), int(ttl.Seconds()), bytes).Result()
+		msg.ID.String(), int(ttl.Seconds()), encoded).Result()
 	if err != nil {
 		return err
 	}
@@ -102,78 +108,110 @@ func (r *RDB) EnqueueUnique(msg *base.TaskMessage, ttl time.Duration) error {
 	return nil
 }
 
-// Dequeue queries given queues in order and pops a task message if there is one and returns it.
+// Dequeue queries given queues in order and pops a task message
+// off a queue if one exists and returns the message and deadline.
+// Dequeue skips a queue if the queue is paused.
 // If all queues are empty, ErrNoProcessableTask error is returned.
-func (r *RDB) Dequeue(qnames ...string) (*base.TaskMessage, error) {
-	var data string
-	var err error
-	if len(qnames) == 1 {
-		data, err = r.dequeueSingle(base.QueueKey(qnames[0]))
-	} else {
-		var keys []string
-		for _, q := range qnames {
-			keys = append(keys, base.QueueKey(q))
-		}
-		data, err = r.dequeue(keys...)
+func (r *RDB) Dequeue(qnames ...string) (msg *base.TaskMessage, deadline time.Time, err error) {
+	var qkeys []interface{}
+	for _, q := range qnames {
+		qkeys = append(qkeys, base.QueueKey(q))
 	}
+	data, d, err := r.dequeue(qkeys...)
 	if err == redis.Nil {
-		return nil, ErrNoProcessableTask
+		return nil, time.Time{}, ErrNoProcessableTask
 	}
 	if err != nil {
-		return nil, err
+		return nil, time.Time{}, err
 	}
-	var msg base.TaskMessage
-	err = json.Unmarshal([]byte(data), &msg)
-	if err != nil {
-		return nil, err
+	if msg, err = base.DecodeMessage(data); err != nil {
+		return nil, time.Time{}, err
 	}
-	return &msg, nil
+	return msg, time.Unix(d, 0), nil
 }
 
-func (r *RDB) dequeueSingle(queue string) (data string, err error) {
-	// timeout needed to avoid blocking forever
-	return r.client.BRPopLPush(queue, base.InProgressQueue, time.Second).Result()
-}
-
-// KEYS[1] -> asynq:in_progress
-// ARGV    -> List of queues to query in order
+// KEYS[1]  -> asynq:in_progress
+// KEYS[2]  -> asynq:paused
+// KEYS[3]  -> asynq:deadlines
+// ARGV[1]  -> current time in Unix time
+// ARGV[2:] -> List of queues to query in order
+//
+// dequeueCmd checks whether a queue is paused first, before
+// calling RPOPLPUSH to pop a task from the queue.
+// It computes the task deadline by inspecting Timout and Deadline fields,
+// and inserts the task with deadlines set.
 var dequeueCmd = redis.NewScript(`
-local res
-for _, qkey in ipairs(ARGV) do
-	res = redis.call("RPOPLPUSH", qkey, KEYS[1])
-	if res then
-		return res
+for i = 2, table.getn(ARGV) do
+	local qkey = ARGV[i]
+	if redis.call("SISMEMBER", KEYS[2], qkey) == 0 then
+		local msg = redis.call("RPOPLPUSH", qkey, KEYS[1])
+		if msg then
+			local decoded = cjson.decode(msg)
+			local timeout = decoded["Timeout"]
+			local deadline = decoded["Deadline"]
+			local score
+			if timeout ~= 0 and deadline ~= 0 then
+				score = math.min(ARGV[1]+timeout, deadline)
+			elseif timeout ~= 0 then
+				score = ARGV[1] + timeout
+			elseif deadline ~= 0 then
+				score = deadline
+			else
+				return redis.error_reply("asynq internal error: both timeout and deadline are not set")
+			end
+			redis.call("ZADD", KEYS[3], score, msg)
+			return {msg, score}
+		end
 	end
 end
-return res`)
+return nil`)
 
-func (r *RDB) dequeue(queues ...string) (data string, err error) {
+func (r *RDB) dequeue(qkeys ...interface{}) (msgjson string, deadline int64, err error) {
 	var args []interface{}
-	for _, qkey := range queues {
-		args = append(args, qkey)
-	}
-	res, err := dequeueCmd.Run(r.client, []string{base.InProgressQueue}, args...).Result()
+	args = append(args, time.Now().Unix())
+	args = append(args, qkeys...)
+	res, err := dequeueCmd.Run(r.client,
+		[]string{base.InProgressQueue, base.PausedQueues, base.KeyDeadlines}, args...).Result()
 	if err != nil {
-		return "", err
+		return "", 0, err
 	}
-	return cast.ToStringE(res)
+	data, err := cast.ToSliceE(res)
+	if err != nil {
+		return "", 0, err
+	}
+	if len(data) != 2 {
+		return "", 0, fmt.Errorf("asynq: internal error: dequeue command returned %d values", len(data))
+	}
+	if msgjson, err = cast.ToStringE(data[0]); err != nil {
+		return "", 0, err
+	}
+	if deadline, err = cast.ToInt64E(data[1]); err != nil {
+		return "", 0, err
+	}
+	return msgjson, deadline, nil
 }
 
 // KEYS[1] -> asynq:in_progress
-// KEYS[2] -> asynq:processed:<yyyy-mm-dd>
-// KEYS[3] -> unique key in the format <type>:<payload>:<qname>
+// KEYS[2] -> asynq:deadlines
+// KEYS[3] -> asynq:processed:<yyyy-mm-dd>
+// KEYS[4] -> unique key in the format <type>:<payload>:<qname>
 // ARGV[1] -> base.TaskMessage value
 // ARGV[2] -> stats expiration timestamp
 // ARGV[3] -> task ID
 // Note: LREM count ZERO means "remove all elements equal to val"
 var doneCmd = redis.NewScript(`
-redis.call("LREM", KEYS[1], 0, ARGV[1]) 
-local n = redis.call("INCR", KEYS[2])
-if tonumber(n) == 1 then
-	redis.call("EXPIREAT", KEYS[2], ARGV[2])
+if redis.call("LREM", KEYS[1], 0, ARGV[1]) == 0 then
+  return redis.error_reply("NOT FOUND")
 end
-if string.len(KEYS[3]) > 0 and redis.call("GET", KEYS[3]) == ARGV[3] then
-  redis.call("DEL", KEYS[3])
+if redis.call("ZREM", KEYS[2], ARGV[1]) == 0 then
+  return redis.error_reply("NOT FOUND")
+end
+local n = redis.call("INCR", KEYS[3])
+if tonumber(n) == 1 then
+	redis.call("EXPIREAT", KEYS[3], ARGV[2])
+end
+if string.len(KEYS[4]) > 0 and redis.call("GET", KEYS[4]) == ARGV[3] then
+  redis.call("DEL", KEYS[4])
 end
 return redis.status_reply("OK")
 `)
@@ -181,7 +219,7 @@ return redis.status_reply("OK")
 // Done removes the task from in-progress queue to mark the task as done.
 // It removes a uniqueness lock acquired by the task, if any.
 func (r *RDB) Done(msg *base.TaskMessage) error {
-	bytes, err := json.Marshal(msg)
+	encoded, err := base.EncodeMessage(msg)
 	if err != nil {
 		return err
 	}
@@ -189,28 +227,34 @@ func (r *RDB) Done(msg *base.TaskMessage) error {
 	processedKey := base.ProcessedKey(now)
 	expireAt := now.Add(statsTTL)
 	return doneCmd.Run(r.client,
-		[]string{base.InProgressQueue, processedKey, msg.UniqueKey},
-		bytes, expireAt.Unix(), msg.ID.String()).Err()
+		[]string{base.InProgressQueue, base.KeyDeadlines, processedKey, msg.UniqueKey},
+		encoded, expireAt.Unix(), msg.ID.String()).Err()
 }
 
 // KEYS[1] -> asynq:in_progress
-// KEYS[2] -> asynq:queues:<qname>
+// KEYS[2] -> asynq:deadlines
+// KEYS[3] -> asynq:queues:<qname>
 // ARGV[1] -> base.TaskMessage value
 // Note: Use RPUSH to push to the head of the queue.
 var requeueCmd = redis.NewScript(`
-redis.call("LREM", KEYS[1], 0, ARGV[1])
-redis.call("RPUSH", KEYS[2], ARGV[1])
+if redis.call("LREM", KEYS[1], 0, ARGV[1]) == 0 then
+  return redis.error_reply("NOT FOUND")
+end
+if redis.call("ZREM", KEYS[2], ARGV[1]) == 0 then
+  return redis.error_reply("NOT FOUND")
+end
+redis.call("RPUSH", KEYS[3], ARGV[1])
 return redis.status_reply("OK")`)
 
 // Requeue moves the task from in-progress queue to the specified queue.
 func (r *RDB) Requeue(msg *base.TaskMessage) error {
-	bytes, err := json.Marshal(msg)
+	encoded, err := base.EncodeMessage(msg)
 	if err != nil {
 		return err
 	}
 	return requeueCmd.Run(r.client,
-		[]string{base.InProgressQueue, base.QueueKey(msg.Queue)},
-		string(bytes)).Err()
+		[]string{base.InProgressQueue, base.KeyDeadlines, base.QueueKey(msg.Queue)},
+		encoded).Err()
 }
 
 // KEYS[1] -> asynq:scheduled
@@ -226,7 +270,7 @@ return 1
 
 // Schedule adds the task to the backlog queue to be processed in the future.
 func (r *RDB) Schedule(msg *base.TaskMessage, processAt time.Time) error {
-	bytes, err := json.Marshal(msg)
+	encoded, err := base.EncodeMessage(msg)
 	if err != nil {
 		return err
 	}
@@ -234,7 +278,7 @@ func (r *RDB) Schedule(msg *base.TaskMessage, processAt time.Time) error {
 	score := float64(processAt.Unix())
 	return scheduleCmd.Run(r.client,
 		[]string{base.ScheduledQueue, base.AllQueues},
-		score, bytes, qkey).Err()
+		score, encoded, qkey).Err()
 }
 
 // KEYS[1] -> unique key in the format <type>:<payload>:<qname>
@@ -258,7 +302,7 @@ return 1
 // ScheduleUnique adds the task to the backlog queue to be processed in the future if the uniqueness lock can be acquired.
 // It returns ErrDuplicateTask if the lock cannot be acquired.
 func (r *RDB) ScheduleUnique(msg *base.TaskMessage, processAt time.Time, ttl time.Duration) error {
-	bytes, err := json.Marshal(msg)
+	encoded, err := base.EncodeMessage(msg)
 	if err != nil {
 		return err
 	}
@@ -266,7 +310,7 @@ func (r *RDB) ScheduleUnique(msg *base.TaskMessage, processAt time.Time, ttl tim
 	score := float64(processAt.Unix())
 	res, err := scheduleUniqueCmd.Run(r.client,
 		[]string{msg.UniqueKey, base.ScheduledQueue, base.AllQueues},
-		msg.ID.String(), int(ttl.Seconds()), score, bytes, qkey).Result()
+		msg.ID.String(), int(ttl.Seconds()), score, encoded, qkey).Result()
 	if err != nil {
 		return err
 	}
@@ -281,37 +325,43 @@ func (r *RDB) ScheduleUnique(msg *base.TaskMessage, processAt time.Time, ttl tim
 }
 
 // KEYS[1] -> asynq:in_progress
-// KEYS[2] -> asynq:retry
-// KEYS[3] -> asynq:processed:<yyyy-mm-dd>
-// KEYS[4] -> asynq:failure:<yyyy-mm-dd>
+// KEYS[2] -> asynq:deadlines
+// KEYS[3] -> asynq:retry
+// KEYS[4] -> asynq:processed:<yyyy-mm-dd>
+// KEYS[5] -> asynq:failure:<yyyy-mm-dd>
 // ARGV[1] -> base.TaskMessage value to remove from base.InProgressQueue queue
 // ARGV[2] -> base.TaskMessage value to add to Retry queue
 // ARGV[3] -> retry_at UNIX timestamp
 // ARGV[4] -> stats expiration timestamp
 var retryCmd = redis.NewScript(`
-redis.call("LREM", KEYS[1], 0, ARGV[1])
-redis.call("ZADD", KEYS[2], ARGV[3], ARGV[2])
-local n = redis.call("INCR", KEYS[3])
-if tonumber(n) == 1 then
-	redis.call("EXPIREAT", KEYS[3], ARGV[4])
+if redis.call("LREM", KEYS[1], 0, ARGV[1]) == 0 then
+  return redis.error_reply("NOT FOUND")
 end
-local m = redis.call("INCR", KEYS[4])
-if tonumber(m) == 1 then
+if redis.call("ZREM", KEYS[2], ARGV[1]) == 0 then
+  return redis.error_reply("NOT FOUND")
+end
+redis.call("ZADD", KEYS[3], ARGV[3], ARGV[2])
+local n = redis.call("INCR", KEYS[4])
+if tonumber(n) == 1 then
 	redis.call("EXPIREAT", KEYS[4], ARGV[4])
 end
+local m = redis.call("INCR", KEYS[5])
+if tonumber(m) == 1 then
+	redis.call("EXPIREAT", KEYS[5], ARGV[4])
+end
 return redis.status_reply("OK")`)
 
 // Retry moves the task from in-progress to retry queue, incrementing retry count
 // and assigning error message to the task message.
 func (r *RDB) Retry(msg *base.TaskMessage, processAt time.Time, errMsg string) error {
-	bytesToRemove, err := json.Marshal(msg)
+	msgToRemove, err := base.EncodeMessage(msg)
 	if err != nil {
 		return err
 	}
 	modified := *msg
 	modified.Retried++
 	modified.ErrorMsg = errMsg
-	bytesToAdd, err := json.Marshal(&modified)
+	msgToAdd, err := base.EncodeMessage(&modified)
 	if err != nil {
 		return err
 	}
@@ -320,8 +370,8 @@ func (r *RDB) Retry(msg *base.TaskMessage, processAt time.Time, errMsg string) e
 	failureKey := base.FailureKey(now)
 	expireAt := now.Add(statsTTL)
 	return retryCmd.Run(r.client,
-		[]string{base.InProgressQueue, base.RetryQueue, processedKey, failureKey},
-		string(bytesToRemove), string(bytesToAdd), processAt.Unix(), expireAt.Unix()).Err()
+		[]string{base.InProgressQueue, base.KeyDeadlines, base.RetryQueue, processedKey, failureKey},
+		msgToRemove, msgToAdd, processAt.Unix(), expireAt.Unix()).Err()
 }
 
 const (
@@ -330,9 +380,10 @@ const (
 )
 
 // KEYS[1] -> asynq:in_progress
-// KEYS[2] -> asynq:dead
-// KEYS[3] -> asynq:processed:<yyyy-mm-dd>
-// KEYS[4] -> asynq.failure:<yyyy-mm-dd>
+// KEYS[2] -> asynq:deadlines
+// KEYS[3] -> asynq:dead
+// KEYS[4] -> asynq:processed:<yyyy-mm-dd>
+// KEYS[5] -> asynq.failure:<yyyy-mm-dd>
 // ARGV[1] -> base.TaskMessage value to remove from base.InProgressQueue queue
 // ARGV[2] -> base.TaskMessage value to add to Dead queue
 // ARGV[3] -> died_at UNIX timestamp
@@ -340,31 +391,36 @@ const (
 // ARGV[5] -> max number of tasks in dead queue (e.g., 100)
 // ARGV[6] -> stats expiration timestamp
 var killCmd = redis.NewScript(`
-redis.call("LREM", KEYS[1], 0, ARGV[1])
-redis.call("ZADD", KEYS[2], ARGV[3], ARGV[2])
-redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", ARGV[4])
-redis.call("ZREMRANGEBYRANK", KEYS[2], 0, -ARGV[5])
-local n = redis.call("INCR", KEYS[3])
-if tonumber(n) == 1 then
-	redis.call("EXPIREAT", KEYS[3], ARGV[6])
+if redis.call("LREM", KEYS[1], 0, ARGV[1]) == 0 then
+  return redis.error_reply("NOT FOUND")
 end
-local m = redis.call("INCR", KEYS[4])
-if tonumber(m) == 1 then
+if redis.call("ZREM", KEYS[2], ARGV[1]) == 0 then
+  return redis.error_reply("NOT FOUND")
+end
+redis.call("ZADD", KEYS[3], ARGV[3], ARGV[2])
+redis.call("ZREMRANGEBYSCORE", KEYS[3], "-inf", ARGV[4])
+redis.call("ZREMRANGEBYRANK", KEYS[3], 0, -ARGV[5])
+local n = redis.call("INCR", KEYS[4])
+if tonumber(n) == 1 then
 	redis.call("EXPIREAT", KEYS[4], ARGV[6])
 end
+local m = redis.call("INCR", KEYS[5])
+if tonumber(m) == 1 then
+	redis.call("EXPIREAT", KEYS[5], ARGV[6])
+end
 return redis.status_reply("OK")`)
 
 // Kill sends the task to "dead" queue from in-progress queue, assigning
 // the error message to the task.
 // It also trims the set by timestamp and set size.
 func (r *RDB) Kill(msg *base.TaskMessage, errMsg string) error {
-	bytesToRemove, err := json.Marshal(msg)
+	msgToRemove, err := base.EncodeMessage(msg)
 	if err != nil {
 		return err
 	}
 	modified := *msg
 	modified.ErrorMsg = errMsg
-	bytesToAdd, err := json.Marshal(&modified)
+	msgToAdd, err := base.EncodeMessage(&modified)
 	if err != nil {
 		return err
 	}
@@ -374,51 +430,21 @@ func (r *RDB) Kill(msg *base.TaskMessage, errMsg string) error {
 	failureKey := base.FailureKey(now)
 	expireAt := now.Add(statsTTL)
 	return killCmd.Run(r.client,
-		[]string{base.InProgressQueue, base.DeadQueue, processedKey, failureKey},
-		string(bytesToRemove), string(bytesToAdd), now.Unix(), limit, maxDeadTasks, expireAt.Unix()).Err()
+		[]string{base.InProgressQueue, base.KeyDeadlines, base.DeadQueue, processedKey, failureKey},
+		msgToRemove, msgToAdd, now.Unix(), limit, maxDeadTasks, expireAt.Unix()).Err()
 }
 
-// KEYS[1] -> asynq:in_progress
-// ARGV[1] -> queue prefix
-var requeueAllCmd = redis.NewScript(`
-local msgs = redis.call("LRANGE", KEYS[1], 0, -1)
-for _, msg in ipairs(msgs) do
-	local decoded = cjson.decode(msg)
-	local qkey = ARGV[1] .. decoded["Queue"]
-	redis.call("RPUSH", qkey, msg)
-	redis.call("LREM", KEYS[1], 0, msg)
-end
-return table.getn(msgs)`)
-
-// RequeueAll moves all tasks from in-progress list to the queue
-// and reports the number of tasks restored.
-func (r *RDB) RequeueAll() (int64, error) {
-	res, err := requeueAllCmd.Run(r.client, []string{base.InProgressQueue}, base.QueuePrefix).Result()
-	if err != nil {
-		return 0, err
-	}
-	n, ok := res.(int64)
-	if !ok {
-		return 0, fmt.Errorf("could not cast %v to int64", res)
-	}
-	return n, nil
-}
-
-// CheckAndEnqueue checks for all scheduled tasks and enqueues any tasks that
-// have to be processed.
-//
-// qnames specifies to which queues to send tasks.
-func (r *RDB) CheckAndEnqueue(qnames ...string) error {
+// CheckAndEnqueue checks for all scheduled/retry tasks and enqueues any tasks that
+// are ready to be processed.
+func (r *RDB) CheckAndEnqueue() (err error) {
 	delayed := []string{base.ScheduledQueue, base.RetryQueue}
 	for _, zset := range delayed {
-		var err error
-		if len(qnames) == 1 {
-			err = r.forwardSingle(zset, base.QueueKey(qnames[0]))
-		} else {
-			err = r.forward(zset)
-		}
-		if err != nil {
-			return err
+		n := 1
+		for n != 0 {
+			n, err = r.forward(zset)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	return nil
@@ -427,53 +453,61 @@ func (r *RDB) CheckAndEnqueue(qnames ...string) error {
 // KEYS[1] -> source queue (e.g. scheduled or retry queue)
 // ARGV[1] -> current unix time
 // ARGV[2] -> queue prefix
+// Note: Script moves tasks up to 100 at a time to keep the runtime of script short.
 var forwardCmd = redis.NewScript(`
-local msgs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1])
+local msgs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, 100)
 for _, msg in ipairs(msgs) do
 	local decoded = cjson.decode(msg)
 	local qkey = ARGV[2] .. decoded["Queue"]
 	redis.call("LPUSH", qkey, msg)
 	redis.call("ZREM", KEYS[1], msg)
 end
-return msgs`)
+return table.getn(msgs)`)
 
-// forward moves all tasks with a score less than the current unix time
-// from the src zset.
-func (r *RDB) forward(src string) error {
+// forward moves tasks with a score less than the current unix time
+// from the src zset. It returns the number of tasks moved.
+func (r *RDB) forward(src string) (int, error) {
 	now := float64(time.Now().Unix())
-	return forwardCmd.Run(r.client,
-		[]string{src}, now, base.QueuePrefix).Err()
+	res, err := forwardCmd.Run(r.client,
+		[]string{src}, now, base.QueuePrefix).Result()
+	if err != nil {
+		return 0, err
+	}
+	return cast.ToInt(res), nil
 }
 
-// KEYS[1] -> source queue (e.g. scheduled or retry queue)
-// KEYS[2] -> destination queue
-var forwardSingleCmd = redis.NewScript(`
-local msgs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1])
-for _, msg in ipairs(msgs) do
-	redis.call("LPUSH", KEYS[2], msg)
-	redis.call("ZREM", KEYS[1], msg)
-end
-return msgs`)
-
-// forwardSingle moves all tasks with a score less than the current unix time
-// from the src zset to dst list.
-func (r *RDB) forwardSingle(src, dst string) error {
-	now := float64(time.Now().Unix())
-	return forwardSingleCmd.Run(r.client,
-		[]string{src, dst}, now).Err()
+// ListDeadlineExceeded returns a list of task messages that have exceeded the given deadline.
+func (r *RDB) ListDeadlineExceeded(deadline time.Time) ([]*base.TaskMessage, error) {
+	var msgs []*base.TaskMessage
+	opt := &redis.ZRangeBy{
+		Min: "-inf",
+		Max: strconv.FormatInt(deadline.Unix(), 10),
+	}
+	res, err := r.client.ZRangeByScore(base.KeyDeadlines, opt).Result()
+	if err != nil {
+		return nil, err
+	}
+	for _, s := range res {
+		msg, err := base.DecodeMessage(s)
+		if err != nil {
+			return nil, err
+		}
+		msgs = append(msgs, msg)
+	}
+	return msgs, nil
 }
 
 // KEYS[1]  -> asynq:servers:<host:pid:sid>
 // KEYS[2]  -> asynq:servers
 // KEYS[3]  -> asynq:workers<host:pid:sid>
-// keys[4]  -> asynq:workers
+// KEYS[4]  -> asynq:workers
 // ARGV[1]  -> expiration time
 // ARGV[2]  -> TTL in seconds
-// ARGV[3]  -> process info
+// ARGV[3]  -> server info
 // ARGV[4:] -> alternate key-value pair of (worker id, worker data)
 // Note: Add key to ZSET with expiration time as score.
 // ref: https://github.com/antirez/redis/issues/135#issuecomment-2361996
-var writeProcessInfoCmd = redis.NewScript(`
+var writeServerStateCmd = redis.NewScript(`
 redis.call("SETEX", KEYS[1], ARGV[2], ARGV[3])
 redis.call("ZADD", KEYS[2], ARGV[1], KEYS[1])
 redis.call("DEL", KEYS[3])
@@ -484,27 +518,24 @@ redis.call("EXPIRE", KEYS[3], ARGV[2])
 redis.call("ZADD", KEYS[4], ARGV[1], KEYS[3])
 return redis.status_reply("OK")`)
 
-// WriteServerState writes server state data to redis with expiration  set to the value ttl.
-func (r *RDB) WriteServerState(ss *base.ServerState, ttl time.Duration) error {
-	info := ss.GetInfo()
+// WriteServerState writes server state data to redis with expiration set to the value ttl.
+func (r *RDB) WriteServerState(info *base.ServerInfo, workers []*base.WorkerInfo, ttl time.Duration) error {
 	bytes, err := json.Marshal(info)
 	if err != nil {
 		return err
 	}
-	var args []interface{} // args to the lua script
 	exp := time.Now().Add(ttl).UTC()
-	workers := ss.GetWorkers()
-	args = append(args, float64(exp.Unix()), ttl.Seconds(), bytes)
+	args := []interface{}{float64(exp.Unix()), ttl.Seconds(), bytes} // args to the lua script
 	for _, w := range workers {
 		bytes, err := json.Marshal(w)
 		if err != nil {
 			continue // skip bad data
 		}
-		args = append(args, w.ID.String(), bytes)
+		args = append(args, w.ID, bytes)
 	}
 	skey := base.ServerInfoKey(info.Host, info.PID, info.ServerID)
 	wkey := base.WorkersKey(info.Host, info.PID, info.ServerID)
-	return writeProcessInfoCmd.Run(r.client,
+	return writeServerStateCmd.Run(r.client,
 		[]string{skey, base.AllServers, wkey, base.AllWorkers},
 		args...).Err()
 }
@@ -513,7 +544,7 @@ func (r *RDB) WriteServerState(ss *base.ServerState, ttl time.Duration) error {
 // KEYS[2] -> asynq:servers:<host:pid:sid>
 // KEYS[3] -> asynq:workers
 // KEYS[4] -> asynq:workers<host:pid:sid>
-var clearProcessInfoCmd = redis.NewScript(`
+var clearServerStateCmd = redis.NewScript(`
 redis.call("ZREM", KEYS[1], KEYS[2])
 redis.call("DEL", KEYS[2])
 redis.call("ZREM", KEYS[3], KEYS[4])
@@ -521,12 +552,10 @@ redis.call("DEL", KEYS[4])
 return redis.status_reply("OK")`)
 
 // ClearServerState deletes server state data from redis.
-func (r *RDB) ClearServerState(ss *base.ServerState) error {
-	info := ss.GetInfo()
-	host, pid, id := info.Host, info.PID, info.ServerID
-	skey := base.ServerInfoKey(host, pid, id)
-	wkey := base.WorkersKey(host, pid, id)
-	return clearProcessInfoCmd.Run(r.client,
+func (r *RDB) ClearServerState(host string, pid int, serverID string) error {
+	skey := base.ServerInfoKey(host, pid, serverID)
+	wkey := base.WorkersKey(host, pid, serverID)
+	return clearServerStateCmd.Run(r.client,
 		[]string{base.AllServers, skey, base.AllWorkers, wkey}).Err()
 }
 

internal/testbroker/testbroker.go

@@ -26,6 +26,9 @@ type TestBroker struct {
 	real base.Broker
 }
 
+// Make sure TestBroker implements Broker interface at compile time.
+var _ base.Broker = (*TestBroker)(nil)
+
 func NewTestBroker(b base.Broker) *TestBroker {
 	return &TestBroker{real: b}
 }
@@ -60,11 +63,11 @@ func (tb *TestBroker) EnqueueUnique(msg *base.TaskMessage, ttl time.Duration) er
 	return tb.real.EnqueueUnique(msg, ttl)
 }
 
-func (tb *TestBroker) Dequeue(qnames ...string) (*base.TaskMessage, error) {
+func (tb *TestBroker) Dequeue(qnames ...string) (*base.TaskMessage, time.Time, error) {
 	tb.mu.Lock()
 	defer tb.mu.Unlock()
 	if tb.sleeping {
-		return nil, errRedisDown
+		return nil, time.Time{}, errRedisDown
 	}
 	return tb.real.Dequeue(qnames...)
 }
@@ -123,16 +126,7 @@ func (tb *TestBroker) Kill(msg *base.TaskMessage, errMsg string) error {
 	return tb.real.Kill(msg, errMsg)
 }
 
-func (tb *TestBroker) RequeueAll() (int64, error) {
-	tb.mu.Lock()
-	defer tb.mu.Unlock()
-	if tb.sleeping {
-		return 0, errRedisDown
-	}
-	return tb.real.RequeueAll()
-}
-
-func (tb *TestBroker) CheckAndEnqueue(qnames ...string) error {
+func (tb *TestBroker) CheckAndEnqueue() error {
 	tb.mu.Lock()
 	defer tb.mu.Unlock()
 	if tb.sleeping {
@@ -141,22 +135,31 @@ func (tb *TestBroker) CheckAndEnqueue(qnames ...string) error {
 	return tb.real.CheckAndEnqueue()
 }
 
-func (tb *TestBroker) WriteServerState(ss *base.ServerState, ttl time.Duration) error {
+func (tb *TestBroker) ListDeadlineExceeded(deadline time.Time) ([]*base.TaskMessage, error) {
 	tb.mu.Lock()
 	defer tb.mu.Unlock()
 	if tb.sleeping {
-		return errRedisDown
+		return nil, errRedisDown
 	}
-	return tb.real.WriteServerState(ss, ttl)
+	return tb.real.ListDeadlineExceeded(deadline)
 }
 
-func (tb *TestBroker) ClearServerState(ss *base.ServerState) error {
+func (tb *TestBroker) WriteServerState(info *base.ServerInfo, workers []*base.WorkerInfo, ttl time.Duration) error {
 	tb.mu.Lock()
 	defer tb.mu.Unlock()
 	if tb.sleeping {
 		return errRedisDown
 	}
-	return tb.real.ClearServerState(ss)
+	return tb.real.WriteServerState(info, workers, ttl)
+}
+
+func (tb *TestBroker) ClearServerState(host string, pid int, serverID string) error {
+	tb.mu.Lock()
+	defer tb.mu.Unlock()
+	if tb.sleeping {
+		return errRedisDown
+	}
+	return tb.real.ClearServerState(host, pid, serverID)
 }
 
 func (tb *TestBroker) CancelationPubSub() (*redis.PubSub, error) {
@@ -177,6 +180,15 @@ func (tb *TestBroker) PublishCancelation(id string) error {
 	return tb.real.PublishCancelation(id)
 }
 
+func (tb *TestBroker) Ping() error {
+	tb.mu.Lock()
+	defer tb.mu.Unlock()
+	if tb.sleeping {
+		return errRedisDown
+	}
+	return tb.real.Ping()
+}
+
 func (tb *TestBroker) Close() error {
 	tb.mu.Lock()
 	defer tb.mu.Unlock()

payload.go

@@ -5,6 +5,7 @@
 package asynq
 
 import (
+	"encoding/json"
 	"fmt"
 	"time"
 
@@ -30,6 +31,19 @@ func (p Payload) Has(key string) bool {
 	return ok
 }
 
+func toInt(v interface{}) (int, error) {
+	switch v := v.(type) {
+	case json.Number:
+		val, err := v.Int64()
+		if err != nil {
+			return 0, err
+		}
+		return int(val), nil
+	default:
+		return cast.ToIntE(v)
+	}
+}
+
 // GetString returns a string value if a string type is associated with
 // the key, otherwise reports an error.
 func (p Payload) GetString(key string) (string, error) {
@@ -47,7 +61,7 @@ func (p Payload) GetInt(key string) (int, error) {
 	if !ok {
 		return 0, &errKeyNotFound{key}
 	}
-	return cast.ToIntE(v)
+	return toInt(v)
 }
 
 // GetFloat64 returns a float64 value if a numeric type is associated with
@@ -57,7 +71,12 @@ func (p Payload) GetFloat64(key string) (float64, error) {
 	if !ok {
 		return 0, &errKeyNotFound{key}
 	}
-	return cast.ToFloat64E(v)
+	switch v := v.(type) {
+	case json.Number:
+		return v.Float64()
+	default:
+		return cast.ToFloat64E(v)
+	}
 }
 
 // GetBool returns a boolean value if a boolean type is associated with
@@ -87,7 +106,20 @@ func (p Payload) GetIntSlice(key string) ([]int, error) {
 	if !ok {
 		return nil, &errKeyNotFound{key}
 	}
-	return cast.ToIntSliceE(v)
+	switch v := v.(type) {
+	case []interface{}:
+		var res []int
+		for _, elem := range v {
+			val, err := toInt(elem)
+			if err != nil {
+				return nil, err
+			}
+			res = append(res, int(val))
+		}
+		return res, nil
+	default:
+		return cast.ToIntSliceE(v)
+	}
 }
 
 // GetStringMap returns a map of string to empty interface
@@ -131,7 +163,20 @@ func (p Payload) GetStringMapInt(key string) (map[string]int, error) {
 	if !ok {
 		return nil, &errKeyNotFound{key}
 	}
-	return cast.ToStringMapIntE(v)
+	switch v := v.(type) {
+	case map[string]interface{}:
+		res := make(map[string]int)
+		for key, val := range v {
+			ival, err := toInt(val)
+			if err != nil {
+				return nil, err
+			}
+			res[key] = ival
+		}
+		return res, nil
+	default:
+		return cast.ToStringMapIntE(v)
+	}
 }
 
 // GetStringMapBool returns a map of string to boolean
@@ -162,5 +207,14 @@ func (p Payload) GetDuration(key string) (time.Duration, error) {
 	if !ok {
 		return 0, &errKeyNotFound{key}
 	}
-	return cast.ToDurationE(v)
+	switch v := v.(type) {
+	case json.Number:
+		val, err := v.Int64()
+		if err != nil {
+			return 0, err
+		}
+		return time.Duration(val), nil
+	default:
+		return cast.ToDurationE(v)
+	}
 }

payload_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	h "github.com/hibiken/asynq/internal/asynqtest"
 	"github.com/hibiken/asynq/internal/base"
 )
@@ -40,12 +41,11 @@ func TestPayloadString(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -85,12 +85,11 @@ func TestPayloadInt(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -130,12 +129,11 @@ func TestPayloadFloat64(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -175,12 +173,11 @@ func TestPayloadBool(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -221,12 +218,11 @@ func TestPayloadStringSlice(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -268,12 +264,11 @@ func TestPayloadIntSlice(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -315,21 +310,28 @@ func TestPayloadStringMap(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
 		payload = Payload{out.Payload}
 		got, err = payload.GetStringMap(tc.key)
-		diff = cmp.Diff(got, tc.data[tc.key])
+		ignoreOpt := cmpopts.IgnoreMapEntries(func(key string, val interface{}) bool {
+			switch val.(type) {
+			case json.Number:
+				return true
+			default:
+				return false
+			}
+		})
+		diff = cmp.Diff(got, tc.data[tc.key], ignoreOpt)
 		if err != nil || diff != "" {
-			t.Errorf("With Marshaling: Payload.GetStringMap(%q) = %v, %v, want %v, nil",
-				tc.key, got, err, tc.data[tc.key])
+			t.Errorf("With Marshaling: Payload.GetStringMap(%q) = %v, %v, want %v, nil;(-want,+got)\n%s",
+				tc.key, got, err, tc.data[tc.key], diff)
 		}
 
 		// access non-existent key.
@@ -362,12 +364,11 @@ func TestPayloadStringMapString(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -413,12 +414,11 @@ func TestPayloadStringMapStringSlice(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -465,12 +465,11 @@ func TestPayloadStringMapInt(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -517,12 +516,11 @@ func TestPayloadStringMapBool(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -564,12 +562,11 @@ func TestPayloadTime(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -611,12 +608,11 @@ func TestPayloadDuration(t *testing.T) {
 
 		// encode and then decode task messsage.
 		in := h.NewTaskMessage("testing", tc.data)
-		b, err := json.Marshal(in)
+		encoded, err := base.EncodeMessage(in)
 		if err != nil {
 			t.Fatal(err)
 		}
-		var out base.TaskMessage
-		err = json.Unmarshal(b, &out)
+		out, err := base.DecodeMessage(encoded)
 		if err != nil {
 			t.Fatal(err)
 		}

processor.go

@@ -22,8 +22,6 @@ type processor struct {
 	logger *log.Logger
 	broker base.Broker
 
-	ss *base.ServerState
-
 	handler Handler
 
 	queueConfig map[string]int
@@ -52,14 +50,17 @@ type processor struct {
 	done chan struct{}
 	once sync.Once
 
-	// abort channel is closed when the shutdown of the "processor" goroutine starts.
-	abort chan struct{}
-
-	// quit channel communicates to the in-flight worker goroutines to stop.
+	// quit channel is closed when the shutdown of the "processor" goroutine starts.
 	quit chan struct{}
 
+	// abort channel communicates to the in-flight worker goroutines to stop.
+	abort chan struct{}
+
 	// cancelations is a set of cancel functions for all in-progress tasks.
 	cancelations *base.Cancelations
+
+	starting chan<- *base.TaskMessage
+	finished chan<- *base.TaskMessage
 }
 
 type retryDelayFunc func(n int, err error, task *Task) time.Duration
@@ -67,38 +68,42 @@ type retryDelayFunc func(n int, err error, task *Task) time.Duration
 type processorParams struct {
 	logger          *log.Logger
 	broker          base.Broker
-	ss              *base.ServerState
 	retryDelayFunc  retryDelayFunc
 	syncCh          chan<- *syncRequest
 	cancelations    *base.Cancelations
+	concurrency     int
+	queues          map[string]int
+	strictPriority  bool
 	errHandler      ErrorHandler
 	shutdownTimeout time.Duration
+	starting        chan<- *base.TaskMessage
+	finished        chan<- *base.TaskMessage
 }
 
 // newProcessor constructs a new processor.
 func newProcessor(params processorParams) *processor {
-	info := params.ss.GetInfo()
-	qcfg := normalizeQueueCfg(info.Queues)
+	queues := normalizeQueues(params.queues)
 	orderedQueues := []string(nil)
-	if info.StrictPriority {
-		orderedQueues = sortByPriority(qcfg)
+	if params.strictPriority {
+		orderedQueues = sortByPriority(queues)
 	}
 	return &processor{
 		logger:         params.logger,
 		broker:         params.broker,
-		ss:             params.ss,
-		queueConfig:    qcfg,
+		queueConfig:    queues,
 		orderedQueues:  orderedQueues,
 		retryDelayFunc: params.retryDelayFunc,
 		syncRequestCh:  params.syncCh,
 		cancelations:   params.cancelations,
 		errLogLimiter:  rate.NewLimiter(rate.Every(3*time.Second), 1),
-		sema:           make(chan struct{}, info.Concurrency),
+		sema:           make(chan struct{}, params.concurrency),
 		done:           make(chan struct{}),
-		abort:          make(chan struct{}),
 		quit:           make(chan struct{}),
+		abort:          make(chan struct{}),
 		errHandler:     params.errHandler,
 		handler:        HandlerFunc(func(ctx context.Context, t *Task) error { return fmt.Errorf("handler not set") }),
+		starting:       params.starting,
+		finished:       params.finished,
 	}
 }
 
@@ -108,7 +113,7 @@ func (p *processor) stop() {
 	p.once.Do(func() {
 		p.logger.Debug("Processor shutting down...")
 		// Unblock if processor is waiting for sema token.
-		close(p.abort)
+		close(p.quit)
 		// Signal the processor goroutine to stop processing tasks
 		// from the queue.
 		p.done <- struct{}{}
@@ -119,26 +124,17 @@ func (p *processor) stop() {
 func (p *processor) terminate() {
 	p.stop()
 
-	time.AfterFunc(p.shutdownTimeout, func() { close(p.quit) })
+	time.AfterFunc(p.shutdownTimeout, func() { close(p.abort) })
+
 	p.logger.Info("Waiting for all workers to finish...")
-
-	// send cancellation signal to all in-progress task handlers
-	for _, cancel := range p.cancelations.GetAll() {
-		cancel()
-	}
-
 	// block until all workers have released the token
 	for i := 0; i < cap(p.sema); i++ {
 		p.sema <- struct{}{}
 	}
 	p.logger.Info("All workers have finished")
-	p.restore() // move any unfinished tasks back to the queue.
 }
 
 func (p *processor) start(wg *sync.WaitGroup) {
-	// NOTE: The call to "restore" needs to complete before starting
-	// the processor goroutine.
-	p.restore()
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
@@ -157,54 +153,66 @@ func (p *processor) start(wg *sync.WaitGroup) {
 // exec pulls a task out of the queue and starts a worker goroutine to
 // process the task.
 func (p *processor) exec() {
-	qnames := p.queues()
-	msg, err := p.broker.Dequeue(qnames...)
-	switch {
-	case err == rdb.ErrNoProcessableTask:
-		// queues are empty, this is a normal behavior.
-		if len(qnames) > 1 {
-			// sleep to avoid slamming redis and let scheduler move tasks into queues.
-			// Note: With multiple queues, we are not using blocking pop operation and
-			// polling queues instead. This adds significant load to redis.
-			time.Sleep(time.Second)
-		}
-		p.logger.Debug("All queues are empty")
-		return
-	case err != nil:
-		if p.errLogLimiter.Allow() {
-			p.logger.Errorf("Dequeue error: %v", err)
-		}
-		return
-	}
-
 	select {
-	case <-p.abort:
-		// shutdown is starting, return immediately after requeuing the message.
-		p.requeue(msg)
+	case <-p.quit:
 		return
 	case p.sema <- struct{}{}: // acquire token
-		p.ss.AddWorkerStats(msg, time.Now())
+		qnames := p.queues()
+		msg, deadline, err := p.broker.Dequeue(qnames...)
+		switch {
+		case err == rdb.ErrNoProcessableTask:
+			p.logger.Debug("All queues are empty")
+			// Queues are empty, this is a normal behavior.
+			// Sleep to avoid slamming redis and let scheduler move tasks into queues.
+			// Note: We are not using blocking pop operation and polling queues instead.
+			// This adds significant load to redis.
+			time.Sleep(time.Second)
+			<-p.sema // release token
+			return
+		case err != nil:
+			if p.errLogLimiter.Allow() {
+				p.logger.Errorf("Dequeue error: %v", err)
+			}
+			<-p.sema // release token
+			return
+		}
+
+		p.starting <- msg
 		go func() {
 			defer func() {
-				p.ss.DeleteWorkerStats(msg)
+				p.finished <- msg
 				<-p.sema // release token
 			}()
 
-			ctx, cancel := createContext(msg)
+			ctx, cancel := createContext(msg, deadline)
 			p.cancelations.Add(msg.ID.String(), cancel)
 			defer func() {
 				cancel()
 				p.cancelations.Delete(msg.ID.String())
 			}()
 
+			// check context before starting a worker goroutine.
+			select {
+			case <-ctx.Done():
+				// already canceled (e.g. deadline exceeded).
+				p.retryOrKill(ctx, msg, ctx.Err())
+				return
+			default:
+			}
+
 			resCh := make(chan error, 1)
-			task := NewTask(msg.Type, msg.Payload)
-			go func() { resCh <- perform(ctx, task, p.handler) }()
+			go func() {
+				resCh <- perform(ctx, NewTask(msg.Type, msg.Payload), p.handler)
+			}()
 
 			select {
-			case <-p.quit:
-				// time is up, quit this worker goroutine.
+			case <-p.abort:
+				// time is up, push the message back to queue and quit this worker goroutine.
 				p.logger.Warnf("Quitting worker. task id=%s", msg.ID)
+				p.requeue(msg)
+				return
+			case <-ctx.Done():
+				p.retryOrKill(ctx, msg, ctx.Err())
 				return
 			case resErr := <-resCh:
 				// Note: One of three things should happen.
@@ -212,82 +220,91 @@ func (p *processor) exec() {
 				// 2) Retry -> Removes the message from InProgress & Adds the message to Retry
 				// 3) Kill  -> Removes the message from InProgress & Adds the message to Dead
 				if resErr != nil {
-					if p.errHandler != nil {
-						p.errHandler.HandleError(task, resErr, msg.Retried, msg.Retry)
-					}
-					if msg.Retried >= msg.Retry {
-						p.kill(msg, resErr)
-					} else {
-						p.retry(msg, resErr)
-					}
+					p.retryOrKill(ctx, msg, resErr)
 					return
 				}
-				p.markAsDone(msg)
+				p.markAsDone(ctx, msg)
 			}
 		}()
 	}
 }
 
-// restore moves all tasks from "in-progress" back to queue
-// to restore all unfinished tasks.
-func (p *processor) restore() {
-	n, err := p.broker.RequeueAll()
-	if err != nil {
-		p.logger.Errorf("Could not restore unfinished tasks: %v", err)
-	}
-	if n > 0 {
-		p.logger.Infof("Restored %d unfinished tasks back to queue", n)
-	}
-}
-
 func (p *processor) requeue(msg *base.TaskMessage) {
 	err := p.broker.Requeue(msg)
 	if err != nil {
 		p.logger.Errorf("Could not push task id=%s back to queue: %v", msg.ID, err)
+	} else {
+		p.logger.Infof("Pushed task id=%s back to queue", msg.ID)
 	}
 }
 
-func (p *processor) markAsDone(msg *base.TaskMessage) {
+func (p *processor) markAsDone(ctx context.Context, msg *base.TaskMessage) {
 	err := p.broker.Done(msg)
 	if err != nil {
-		errMsg := fmt.Sprintf("Could not remove task id=%s from %q", msg.ID, base.InProgressQueue)
+		errMsg := fmt.Sprintf("Could not remove task id=%s type=%q from %q err: %+v", msg.ID, msg.Type, base.InProgressQueue, err)
+		deadline, ok := ctx.Deadline()
+		if !ok {
+			panic("asynq: internal error: missing deadline in context")
+		}
 		p.logger.Warnf("%s; Will retry syncing", errMsg)
 		p.syncRequestCh <- &syncRequest{
 			fn: func() error {
 				return p.broker.Done(msg)
 			},
-			errMsg: errMsg,
+			errMsg:   errMsg,
+			deadline: deadline,
 		}
 	}
 }
 
-func (p *processor) retry(msg *base.TaskMessage, e error) {
+func (p *processor) retryOrKill(ctx context.Context, msg *base.TaskMessage, err error) {
+	if p.errHandler != nil {
+		p.errHandler.HandleError(ctx, NewTask(msg.Type, msg.Payload), err)
+	}
+	if msg.Retried >= msg.Retry {
+		p.logger.Warnf("Retry exhausted for task id=%s", msg.ID)
+		p.kill(ctx, msg, err)
+	} else {
+		p.retry(ctx, msg, err)
+	}
+}
+
+func (p *processor) retry(ctx context.Context, msg *base.TaskMessage, e error) {
 	d := p.retryDelayFunc(msg.Retried, e, NewTask(msg.Type, msg.Payload))
 	retryAt := time.Now().Add(d)
 	err := p.broker.Retry(msg, retryAt, e.Error())
 	if err != nil {
 		errMsg := fmt.Sprintf("Could not move task id=%s from %q to %q", msg.ID, base.InProgressQueue, base.RetryQueue)
+		deadline, ok := ctx.Deadline()
+		if !ok {
+			panic("asynq: internal error: missing deadline in context")
+		}
 		p.logger.Warnf("%s; Will retry syncing", errMsg)
 		p.syncRequestCh <- &syncRequest{
 			fn: func() error {
 				return p.broker.Retry(msg, retryAt, e.Error())
 			},
-			errMsg: errMsg,
+			errMsg:   errMsg,
+			deadline: deadline,
 		}
 	}
 }
 
-func (p *processor) kill(msg *base.TaskMessage, e error) {
-	p.logger.Warnf("Retry exhausted for task id=%s", msg.ID)
+func (p *processor) kill(ctx context.Context, msg *base.TaskMessage, e error) {
 	err := p.broker.Kill(msg, e.Error())
 	if err != nil {
 		errMsg := fmt.Sprintf("Could not move task id=%s from %q to %q", msg.ID, base.InProgressQueue, base.DeadQueue)
+		deadline, ok := ctx.Deadline()
+		if !ok {
+			panic("asynq: internal error: missing deadline in context")
+		}
 		p.logger.Warnf("%s; Will retry syncing", errMsg)
 		p.syncRequestCh <- &syncRequest{
 			fn: func() error {
 				return p.broker.Kill(msg, e.Error())
 			},
-			errMsg: errMsg,
+			errMsg:   errMsg,
+			deadline: deadline,
 		}
 	}
 }
@@ -374,16 +391,15 @@ func (x byPriority) Len() int           { return len(x) }
 func (x byPriority) Less(i, j int) bool { return x[i].priority < x[j].priority }
 func (x byPriority) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
 
-// normalizeQueueCfg divides priority numbers by their
-// greatest common divisor.
-func normalizeQueueCfg(queueCfg map[string]int) map[string]int {
+// normalizeQueues divides priority numbers by their greatest common divisor.
+func normalizeQueues(queues map[string]int) map[string]int {
 	var xs []int
-	for _, x := range queueCfg {
+	for _, x := range queues {
 		xs = append(xs, x)
 	}
 	d := gcd(xs...)
 	res := make(map[string]int)
-	for q, x := range queueCfg {
+	for q, x := range queues {
 		res[q] = x / d
 	}
 	return res

processor_test.go

@@ -19,6 +19,29 @@ import (
 	"github.com/hibiken/asynq/internal/rdb"
 )
 
+// fakeHeartbeater receives from starting and finished channels and do nothing.
+func fakeHeartbeater(starting, finished <-chan *base.TaskMessage, done <-chan struct{}) {
+	for {
+		select {
+		case <-starting:
+		case <-finished:
+		case <-done:
+			return
+		}
+	}
+}
+
+// fakeSyncer receives from sync channel and do nothing.
+func fakeSyncer(syncCh <-chan *syncRequest, done <-chan struct{}) {
+	for {
+		select {
+		case <-syncCh:
+		case <-done:
+			return
+		}
+	}
+}
+
 func TestProcessorSuccess(t *testing.T) {
 	r := setup(t)
 	rdbClient := rdb.NewRDB(r)
@@ -63,16 +86,26 @@ func TestProcessorSuccess(t *testing.T) {
 			processed = append(processed, task)
 			return nil
 		}
-		ss := base.NewServerState("localhost", 1234, 10, defaultQueueConfig, false)
+		starting := make(chan *base.TaskMessage)
+		finished := make(chan *base.TaskMessage)
+		syncCh := make(chan *syncRequest)
+		done := make(chan struct{})
+		defer func() { close(done) }()
+		go fakeHeartbeater(starting, finished, done)
+		go fakeSyncer(syncCh, done)
 		p := newProcessor(processorParams{
 			logger:          testLogger,
 			broker:          rdbClient,
-			ss:              ss,
 			retryDelayFunc:  defaultDelayFunc,
-			syncCh:          nil,
+			syncCh:          syncCh,
 			cancelations:    base.NewCancelations(),
+			concurrency:     10,
+			queues:          defaultQueueConfig,
+			strictPriority:  false,
 			errHandler:      nil,
 			shutdownTimeout: defaultShutdownTimeout,
+			starting:        starting,
+			finished:        finished,
 		})
 		p.handler = HandlerFunc(handler)
 
@@ -84,16 +117,90 @@ func TestProcessorSuccess(t *testing.T) {
 				t.Fatal(err)
 			}
 		}
-		time.Sleep(time.Second) // wait for one second to allow all enqueued tasks to be processed.
-		p.terminate()
-
-		if diff := cmp.Diff(tc.wantProcessed, processed, sortTaskOpt, cmp.AllowUnexported(Payload{})); diff != "" {
-			t.Errorf("mismatch found in processed tasks; (-want, +got)\n%s", diff)
-		}
-
+		time.Sleep(2 * time.Second) // wait for two second to allow all enqueued tasks to be processed.
 		if l := r.LLen(base.InProgressQueue).Val(); l != 0 {
 			t.Errorf("%q has %d tasks, want 0", base.InProgressQueue, l)
 		}
+		p.terminate()
+
+		mu.Lock()
+		if diff := cmp.Diff(tc.wantProcessed, processed, sortTaskOpt, cmp.AllowUnexported(Payload{})); diff != "" {
+			t.Errorf("mismatch found in processed tasks; (-want, +got)\n%s", diff)
+		}
+		mu.Unlock()
+	}
+}
+
+// https://github.com/hibiken/asynq/issues/166
+func TestProcessTasksWithLargeNumberInPayload(t *testing.T) {
+	r := setup(t)
+	rdbClient := rdb.NewRDB(r)
+
+	m1 := h.NewTaskMessage("large_number", map[string]interface{}{"data": 111111111111111111})
+	t1 := NewTask(m1.Type, m1.Payload)
+
+	tests := []struct {
+		enqueued      []*base.TaskMessage // initial default queue state
+		wantProcessed []*Task             // tasks to be processed at the end
+	}{
+		{
+			enqueued:      []*base.TaskMessage{m1},
+			wantProcessed: []*Task{t1},
+		},
+	}
+
+	for _, tc := range tests {
+		h.FlushDB(t, r)                        // clean up db before each test case.
+		h.SeedEnqueuedQueue(t, r, tc.enqueued) // initialize default queue.
+
+		var mu sync.Mutex
+		var processed []*Task
+		handler := func(ctx context.Context, task *Task) error {
+			mu.Lock()
+			defer mu.Unlock()
+			if data, err := task.Payload.GetInt("data"); err != nil {
+				t.Errorf("coult not get data from payload: %v", err)
+			} else {
+				t.Logf("data == %d", data)
+			}
+			processed = append(processed, task)
+			return nil
+		}
+		starting := make(chan *base.TaskMessage)
+		finished := make(chan *base.TaskMessage)
+		syncCh := make(chan *syncRequest)
+		done := make(chan struct{})
+		defer func() { close(done) }()
+		go fakeHeartbeater(starting, finished, done)
+		go fakeSyncer(syncCh, done)
+		p := newProcessor(processorParams{
+			logger:          testLogger,
+			broker:          rdbClient,
+			retryDelayFunc:  defaultDelayFunc,
+			syncCh:          syncCh,
+			cancelations:    base.NewCancelations(),
+			concurrency:     10,
+			queues:          defaultQueueConfig,
+			strictPriority:  false,
+			errHandler:      nil,
+			shutdownTimeout: defaultShutdownTimeout,
+			starting:        starting,
+			finished:        finished,
+		})
+		p.handler = HandlerFunc(handler)
+
+		p.start(&sync.WaitGroup{})
+		time.Sleep(2 * time.Second) // wait for two second to allow all enqueued tasks to be processed.
+		if l := r.LLen(base.InProgressQueue).Val(); l != 0 {
+			t.Errorf("%q has %d tasks, want 0", base.InProgressQueue, l)
+		}
+		p.terminate()
+
+		mu.Lock()
+		if diff := cmp.Diff(tc.wantProcessed, processed, sortTaskOpt, cmpopts.IgnoreUnexported(Payload{})); diff != "" {
+			t.Errorf("mismatch found in processed tasks; (-want, +got)\n%s", diff)
+		}
+		mu.Unlock()
 	}
 }
 
@@ -108,19 +215,6 @@ func TestProcessorRetry(t *testing.T) {
 	m4 := h.NewTaskMessage("sync", nil)
 
 	errMsg := "something went wrong"
-	// r* is m* after retry
-	r1 := *m1
-	r1.ErrorMsg = errMsg
-	r2 := *m2
-	r2.ErrorMsg = errMsg
-	r2.Retried = m2.Retried + 1
-	r3 := *m3
-	r3.ErrorMsg = errMsg
-	r3.Retried = m3.Retried + 1
-	r4 := *m4
-	r4.ErrorMsg = errMsg
-	r4.Retried = m4.Retried + 1
-
 	now := time.Now()
 
 	tests := []struct {
@@ -129,7 +223,7 @@ func TestProcessorRetry(t *testing.T) {
 		delay        time.Duration       // retry delay duration
 		handler      Handler             // task handler
 		wait         time.Duration       // wait duration between starting and stopping processor for this test case
-		wantRetry    []h.ZSetEntry       // tasks in retry queue at the end
+		wantRetry    []base.Z            // tasks in retry queue at the end
 		wantDead     []*base.TaskMessage // tasks in dead queue at the end
 		wantErrCount int                 // number of times error handler should be called
 	}{
@@ -140,13 +234,13 @@ func TestProcessorRetry(t *testing.T) {
 			handler: HandlerFunc(func(ctx context.Context, task *Task) error {
 				return fmt.Errorf(errMsg)
 			}),
-			wait: time.Second,
-			wantRetry: []h.ZSetEntry{
-				{Msg: &r2, Score: float64(now.Add(time.Minute).Unix())},
-				{Msg: &r3, Score: float64(now.Add(time.Minute).Unix())},
-				{Msg: &r4, Score: float64(now.Add(time.Minute).Unix())},
+			wait: 2 * time.Second,
+			wantRetry: []base.Z{
+				{Message: h.TaskMessageAfterRetry(*m2, errMsg), Score: now.Add(time.Minute).Unix()},
+				{Message: h.TaskMessageAfterRetry(*m3, errMsg), Score: now.Add(time.Minute).Unix()},
+				{Message: h.TaskMessageAfterRetry(*m4, errMsg), Score: now.Add(time.Minute).Unix()},
 			},
-			wantDead:     []*base.TaskMessage{&r1},
+			wantDead:     []*base.TaskMessage{h.TaskMessageWithError(*m1, errMsg)},
 			wantErrCount: 4,
 		},
 	}
@@ -163,21 +257,29 @@ func TestProcessorRetry(t *testing.T) {
 			mu sync.Mutex // guards n
 			n  int        // number of times error handler is called
 		)
-		errHandler := func(t *Task, err error, retried, maxRetry int) {
+		errHandler := func(ctx context.Context, t *Task, err error) {
 			mu.Lock()
 			defer mu.Unlock()
 			n++
 		}
-		ss := base.NewServerState("localhost", 1234, 10, defaultQueueConfig, false)
+		starting := make(chan *base.TaskMessage)
+		finished := make(chan *base.TaskMessage)
+		done := make(chan struct{})
+		defer func() { close(done) }()
+		go fakeHeartbeater(starting, finished, done)
 		p := newProcessor(processorParams{
 			logger:          testLogger,
 			broker:          rdbClient,
-			ss:              ss,
 			retryDelayFunc:  delayFunc,
 			syncCh:          nil,
 			cancelations:    base.NewCancelations(),
+			concurrency:     10,
+			queues:          defaultQueueConfig,
+			strictPriority:  false,
 			errHandler:      ErrorHandlerFunc(errHandler),
 			shutdownTimeout: defaultShutdownTimeout,
+			starting:        starting,
+			finished:        finished,
 		})
 		p.handler = tc.handler
 
@@ -189,7 +291,7 @@ func TestProcessorRetry(t *testing.T) {
 				t.Fatal(err)
 			}
 		}
-		time.Sleep(tc.wait)
+		time.Sleep(tc.wait) // FIXME: This makes test flaky.
 		p.terminate()
 
 		cmpOpt := cmpopts.EquateApprox(0, float64(time.Second)) // allow up to a second difference in zset score
@@ -241,16 +343,24 @@ func TestProcessorQueues(t *testing.T) {
 	}
 
 	for _, tc := range tests {
-		ss := base.NewServerState("localhost", 1234, 10, tc.queueCfg, false)
+		starting := make(chan *base.TaskMessage)
+		finished := make(chan *base.TaskMessage)
+		done := make(chan struct{})
+		defer func() { close(done) }()
+		go fakeHeartbeater(starting, finished, done)
 		p := newProcessor(processorParams{
 			logger:          testLogger,
 			broker:          nil,
-			ss:              ss,
 			retryDelayFunc:  defaultDelayFunc,
 			syncCh:          nil,
 			cancelations:    base.NewCancelations(),
+			concurrency:     10,
+			queues:          tc.queueCfg,
+			strictPriority:  false,
 			errHandler:      nil,
 			shutdownTimeout: defaultShutdownTimeout,
+			starting:        starting,
+			finished:        finished,
 		})
 		got := p.queues()
 		if diff := cmp.Diff(tc.want, got, sortOpt); diff != "" {
@@ -316,17 +426,24 @@ func TestProcessorWithStrictPriority(t *testing.T) {
 			base.DefaultQueueName: 2,
 			"low":                 1,
 		}
-		// Note: Set concurrency to 1 to make sure tasks are processed one at a time.
-		ss := base.NewServerState("localhost", 1234, 1 /* concurrency */, queueCfg, true /*strict*/)
+		starting := make(chan *base.TaskMessage)
+		finished := make(chan *base.TaskMessage)
+		done := make(chan struct{})
+		defer func() { close(done) }()
+		go fakeHeartbeater(starting, finished, done)
 		p := newProcessor(processorParams{
 			logger:          testLogger,
 			broker:          rdbClient,
-			ss:              ss,
 			retryDelayFunc:  defaultDelayFunc,
 			syncCh:          nil,
 			cancelations:    base.NewCancelations(),
+			concurrency:     1, // Set concurrency to 1 to make sure tasks are processed one at a time.
+			queues:          queueCfg,
+			strictPriority:  true,
 			errHandler:      nil,
 			shutdownTimeout: defaultShutdownTimeout,
+			starting:        starting,
+			finished:        finished,
 		})
 		p.handler = HandlerFunc(handler)
 
@@ -412,7 +529,7 @@ func TestGCD(t *testing.T) {
 	}
 }
 
-func TestNormalizeQueueCfg(t *testing.T) {
+func TestNormalizeQueues(t *testing.T) {
 	tests := []struct {
 		input map[string]int
 		want  map[string]int
@@ -462,9 +579,9 @@ func TestNormalizeQueueCfg(t *testing.T) {
 	}
 
 	for _, tc := range tests {
-		got := normalizeQueueCfg(tc.input)
+		got := normalizeQueues(tc.input)
 		if diff := cmp.Diff(tc.want, got); diff != "" {
-			t.Errorf("normalizeQueueCfg(%v) = %v, want %v; (-want, +got):\n%s",
+			t.Errorf("normalizeQueues(%v) = %v, want %v; (-want, +got):\n%s",
 				tc.input, got, tc.want, diff)
 		}
 	}

recoverer.go

@@ -0,0 +1,96 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package asynq
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/hibiken/asynq/internal/base"
+	"github.com/hibiken/asynq/internal/log"
+)
+
+type recoverer struct {
+	logger         *log.Logger
+	broker         base.Broker
+	retryDelayFunc retryDelayFunc
+
+	// channel to communicate back to the long running "recoverer" goroutine.
+	done chan struct{}
+
+	// poll interval.
+	interval time.Duration
+}
+
+type recovererParams struct {
+	logger         *log.Logger
+	broker         base.Broker
+	interval       time.Duration
+	retryDelayFunc retryDelayFunc
+}
+
+func newRecoverer(params recovererParams) *recoverer {
+	return &recoverer{
+		logger:         params.logger,
+		broker:         params.broker,
+		done:           make(chan struct{}),
+		interval:       params.interval,
+		retryDelayFunc: params.retryDelayFunc,
+	}
+}
+
+func (r *recoverer) terminate() {
+	r.logger.Debug("Recoverer shutting down...")
+	// Signal the recoverer goroutine to stop polling.
+	r.done <- struct{}{}
+}
+
+func (r *recoverer) start(wg *sync.WaitGroup) {
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		timer := time.NewTimer(r.interval)
+		for {
+			select {
+			case <-r.done:
+				r.logger.Debug("Recoverer done")
+				timer.Stop()
+				return
+			case <-timer.C:
+				// Get all tasks which have expired 30 seconds ago or earlier.
+				deadline := time.Now().Add(-30 * time.Second)
+				msgs, err := r.broker.ListDeadlineExceeded(deadline)
+				if err != nil {
+					r.logger.Warn("recoverer: could not list deadline exceeded tasks")
+					continue
+				}
+				const errMsg = "deadline exceeded" // TODO: better error message
+				for _, msg := range msgs {
+					if msg.Retried >= msg.Retry {
+						r.kill(msg, errMsg)
+					} else {
+						r.retry(msg, errMsg)
+					}
+				}
+
+			}
+		}
+	}()
+}
+
+func (r *recoverer) retry(msg *base.TaskMessage, errMsg string) {
+	delay := r.retryDelayFunc(msg.Retried, fmt.Errorf(errMsg), NewTask(msg.Type, msg.Payload))
+	retryAt := time.Now().Add(delay)
+	if err := r.broker.Retry(msg, retryAt, errMsg); err != nil {
+		r.logger.Warnf("recoverer: could not retry deadline exceeded task: %v", err)
+	}
+}
+
+func (r *recoverer) kill(msg *base.TaskMessage, errMsg string) {
+	if err := r.broker.Kill(msg, errMsg); err != nil {
+		r.logger.Warnf("recoverer: could not move task to dead queue: %v", err)
+	}
+}

recoverer_test.go

@@ -0,0 +1,162 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package asynq
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	h "github.com/hibiken/asynq/internal/asynqtest"
+	"github.com/hibiken/asynq/internal/base"
+	"github.com/hibiken/asynq/internal/rdb"
+)
+
+func TestRecoverer(t *testing.T) {
+	r := setup(t)
+	rdbClient := rdb.NewRDB(r)
+
+	t1 := h.NewTaskMessage("task1", nil)
+	t2 := h.NewTaskMessage("task2", nil)
+	t3 := h.NewTaskMessageWithQueue("task3", nil, "critical")
+	t4 := h.NewTaskMessage("task4", nil)
+	t4.Retried = t4.Retry // t4 has reached its max retry count
+
+	now := time.Now()
+	oneHourFromNow := now.Add(1 * time.Hour)
+	fiveMinutesFromNow := now.Add(5 * time.Minute)
+	fiveMinutesAgo := now.Add(-5 * time.Minute)
+	oneHourAgo := now.Add(-1 * time.Hour)
+
+	tests := []struct {
+		desc           string
+		inProgress     []*base.TaskMessage
+		deadlines      []base.Z
+		retry          []base.Z
+		dead           []base.Z
+		wantInProgress []*base.TaskMessage
+		wantDeadlines  []base.Z
+		wantRetry      []*base.TaskMessage
+		wantDead       []*base.TaskMessage
+	}{
+		{
+			desc:       "with one task in-progress",
+			inProgress: []*base.TaskMessage{t1},
+			deadlines: []base.Z{
+				{Message: t1, Score: fiveMinutesAgo.Unix()},
+			},
+			retry:          []base.Z{},
+			dead:           []base.Z{},
+			wantInProgress: []*base.TaskMessage{},
+			wantDeadlines:  []base.Z{},
+			wantRetry: []*base.TaskMessage{
+				h.TaskMessageAfterRetry(*t1, "deadline exceeded"),
+			},
+			wantDead: []*base.TaskMessage{},
+		},
+		{
+			desc:       "with a task with max-retry reached",
+			inProgress: []*base.TaskMessage{t4},
+			deadlines: []base.Z{
+				{Message: t4, Score: fiveMinutesAgo.Unix()},
+			},
+			retry:          []base.Z{},
+			dead:           []base.Z{},
+			wantInProgress: []*base.TaskMessage{},
+			wantDeadlines:  []base.Z{},
+			wantRetry:      []*base.TaskMessage{},
+			wantDead:       []*base.TaskMessage{h.TaskMessageWithError(*t4, "deadline exceeded")},
+		},
+		{
+			desc:       "with multiple tasks in-progress, and one expired",
+			inProgress: []*base.TaskMessage{t1, t2, t3},
+			deadlines: []base.Z{
+				{Message: t1, Score: oneHourAgo.Unix()},
+				{Message: t2, Score: fiveMinutesFromNow.Unix()},
+				{Message: t3, Score: oneHourFromNow.Unix()},
+			},
+			retry:          []base.Z{},
+			dead:           []base.Z{},
+			wantInProgress: []*base.TaskMessage{t2, t3},
+			wantDeadlines: []base.Z{
+				{Message: t2, Score: fiveMinutesFromNow.Unix()},
+				{Message: t3, Score: oneHourFromNow.Unix()},
+			},
+			wantRetry: []*base.TaskMessage{
+				h.TaskMessageAfterRetry(*t1, "deadline exceeded"),
+			},
+			wantDead: []*base.TaskMessage{},
+		},
+		{
+			desc:       "with multiple expired tasks in-progress",
+			inProgress: []*base.TaskMessage{t1, t2, t3},
+			deadlines: []base.Z{
+				{Message: t1, Score: oneHourAgo.Unix()},
+				{Message: t2, Score: fiveMinutesAgo.Unix()},
+				{Message: t3, Score: oneHourFromNow.Unix()},
+			},
+			retry:          []base.Z{},
+			dead:           []base.Z{},
+			wantInProgress: []*base.TaskMessage{t3},
+			wantDeadlines: []base.Z{
+				{Message: t3, Score: oneHourFromNow.Unix()},
+			},
+			wantRetry: []*base.TaskMessage{
+				h.TaskMessageAfterRetry(*t1, "deadline exceeded"),
+				h.TaskMessageAfterRetry(*t2, "deadline exceeded"),
+			},
+			wantDead: []*base.TaskMessage{},
+		},
+		{
+			desc:           "with empty in-progress queue",
+			inProgress:     []*base.TaskMessage{},
+			deadlines:      []base.Z{},
+			retry:          []base.Z{},
+			dead:           []base.Z{},
+			wantInProgress: []*base.TaskMessage{},
+			wantDeadlines:  []base.Z{},
+			wantRetry:      []*base.TaskMessage{},
+			wantDead:       []*base.TaskMessage{},
+		},
+	}
+
+	for _, tc := range tests {
+		h.FlushDB(t, r)
+		h.SeedInProgressQueue(t, r, tc.inProgress)
+		h.SeedDeadlines(t, r, tc.deadlines)
+		h.SeedRetryQueue(t, r, tc.retry)
+		h.SeedDeadQueue(t, r, tc.dead)
+
+		recoverer := newRecoverer(recovererParams{
+			logger:         testLogger,
+			broker:         rdbClient,
+			interval:       1 * time.Second,
+			retryDelayFunc: func(n int, err error, task *Task) time.Duration { return 30 * time.Second },
+		})
+
+		var wg sync.WaitGroup
+		recoverer.start(&wg)
+		time.Sleep(2 * time.Second)
+		recoverer.terminate()
+
+		gotInProgress := h.GetInProgressMessages(t, r)
+		if diff := cmp.Diff(tc.wantInProgress, gotInProgress, h.SortMsgOpt); diff != "" {
+			t.Errorf("%s; mismatch found in %q; (-want,+got)\n%s", tc.desc, base.InProgressQueue, diff)
+		}
+		gotDeadlines := h.GetDeadlinesEntries(t, r)
+		if diff := cmp.Diff(tc.wantDeadlines, gotDeadlines, h.SortZSetEntryOpt); diff != "" {
+			t.Errorf("%s; mismatch found in %q; (-want,+got)\n%s", tc.desc, base.KeyDeadlines, diff)
+		}
+		gotRetry := h.GetRetryMessages(t, r)
+		if diff := cmp.Diff(tc.wantRetry, gotRetry, h.SortMsgOpt); diff != "" {
+			t.Errorf("%s; mismatch found in %q: (-want, +got)\n%s", tc.desc, base.RetryQueue, diff)
+		}
+		gotDead := h.GetDeadMessages(t, r)
+		if diff := cmp.Diff(tc.wantDead, gotDead, h.SortMsgOpt); diff != "" {
+			t.Errorf("%s; mismatch found in %q: (-want, +got)\n%s", tc.desc, base.DeadQueue, diff)
+		}
+	}
+}

scheduler.go

@@ -21,29 +21,20 @@ type scheduler struct {
 
 	// poll interval on average
 	avgInterval time.Duration
-
-	// list of queues to move the tasks into.
-	qnames []string
 }
 
 type schedulerParams struct {
 	logger   *log.Logger
 	broker   base.Broker
 	interval time.Duration
-	queues   map[string]int
 }
 
 func newScheduler(params schedulerParams) *scheduler {
-	var qnames []string
-	for q := range params.queues {
-		qnames = append(qnames, q)
-	}
 	return &scheduler{
 		logger:      params.logger,
 		broker:      params.broker,
 		done:        make(chan struct{}),
 		avgInterval: params.interval,
-		qnames:      qnames,
 	}
 }
 
@@ -71,7 +62,7 @@ func (s *scheduler) start(wg *sync.WaitGroup) {
 }
 
 func (s *scheduler) exec() {
-	if err := s.broker.CheckAndEnqueue(s.qnames...); err != nil {
+	if err := s.broker.CheckAndEnqueue(); err != nil {
 		s.logger.Errorf("Could not enqueue scheduled tasks: %v", err)
 	}
 }

scheduler_test.go

@@ -23,7 +23,6 @@ func TestScheduler(t *testing.T) {
 		logger:   testLogger,
 		broker:   rdbClient,
 		interval: pollInterval,
-		queues:   defaultQueueConfig,
 	})
 	t1 := h.NewTaskMessage("gen_thumbnail", nil)
 	t2 := h.NewTaskMessage("send_email", nil)
@@ -32,8 +31,8 @@ func TestScheduler(t *testing.T) {
 	now := time.Now()
 
 	tests := []struct {
-		initScheduled []h.ZSetEntry       // scheduled queue initial state
-		initRetry     []h.ZSetEntry       // retry queue initial state
+		initScheduled []base.Z            // scheduled queue initial state
+		initRetry     []base.Z            // retry queue initial state
 		initQueue     []*base.TaskMessage // default queue initial state
 		wait          time.Duration       // wait duration before checking for final state
 		wantScheduled []*base.TaskMessage // schedule queue final state
@@ -41,12 +40,12 @@ func TestScheduler(t *testing.T) {
 		wantQueue     []*base.TaskMessage // default queue final state
 	}{
 		{
-			initScheduled: []h.ZSetEntry{
-				{Msg: t1, Score: float64(now.Add(time.Hour).Unix())},
-				{Msg: t2, Score: float64(now.Add(-2 * time.Second).Unix())},
+			initScheduled: []base.Z{
+				{Message: t1, Score: now.Add(time.Hour).Unix()},
+				{Message: t2, Score: now.Add(-2 * time.Second).Unix()},
 			},
-			initRetry: []h.ZSetEntry{
-				{Msg: t3, Score: float64(time.Now().Add(-500 * time.Millisecond).Unix())},
+			initRetry: []base.Z{
+				{Message: t3, Score: time.Now().Add(-500 * time.Millisecond).Unix()},
 			},
 			initQueue:     []*base.TaskMessage{t4},
 			wait:          pollInterval * 2,
@@ -55,12 +54,12 @@ func TestScheduler(t *testing.T) {
 			wantQueue:     []*base.TaskMessage{t2, t3, t4},
 		},
 		{
-			initScheduled: []h.ZSetEntry{
-				{Msg: t1, Score: float64(now.Unix())},
-				{Msg: t2, Score: float64(now.Add(-2 * time.Second).Unix())},
-				{Msg: t3, Score: float64(now.Add(-500 * time.Millisecond).Unix())},
+			initScheduled: []base.Z{
+				{Message: t1, Score: now.Unix()},
+				{Message: t2, Score: now.Add(-2 * time.Second).Unix()},
+				{Message: t3, Score: now.Add(-500 * time.Millisecond).Unix()},
 			},
-			initRetry:     []h.ZSetEntry{},
+			initRetry:     []base.Z{},
 			initQueue:     []*base.TaskMessage{t4},
 			wait:          pollInterval * 2,
 			wantScheduled: []*base.TaskMessage{},

server.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"math"
 	"math/rand"
-	"os"
 	"runtime"
 	"strings"
 	"sync"
@@ -34,19 +33,21 @@ import (
 // (e.g., queue size reaches a certain limit, or the task has been in the
 // queue for a certain amount of time).
 type Server struct {
-	ss *base.ServerState
-
 	logger *log.Logger
 
 	broker base.Broker
 
+	status *base.ServerStatus
+
 	// wait group to wait for all goroutines to finish.
-	wg          sync.WaitGroup
-	scheduler   *scheduler
-	processor   *processor
-	syncer      *syncer
-	heartbeater *heartbeater
-	subscriber  *subscriber
+	wg            sync.WaitGroup
+	scheduler     *scheduler
+	processor     *processor
+	syncer        *syncer
+	heartbeater   *heartbeater
+	subscriber    *subscriber
+	recoverer     *recoverer
+	healthchecker *healthchecker
 }
 
 // Config specifies the server's background-task processing behavior.
@@ -123,20 +124,29 @@ type Config struct {
 	//
 	// If unset or zero, default timeout of 8 seconds is used.
 	ShutdownTimeout time.Duration
+
+	// HealthCheckFunc is called periodically with any errors encountered during ping to the
+	// connected redis server.
+	HealthCheckFunc func(error)
+
+	// HealthCheckInterval specifies the interval between healthchecks.
+	//
+	// If unset or zero, the interval is set to 15 seconds.
+	HealthCheckInterval time.Duration
 }
 
-// An ErrorHandler handles errors returned by the task handler.
+// An ErrorHandler handles an error occured during task processing.
 type ErrorHandler interface {
-	HandleError(task *Task, err error, retried, maxRetry int)
+	HandleError(ctx context.Context, task *Task, err error)
 }
 
 // The ErrorHandlerFunc type is an adapter to allow the use of  ordinary functions as a ErrorHandler.
 // If f is a function with the appropriate signature, ErrorHandlerFunc(f) is a ErrorHandler that calls f.
-type ErrorHandlerFunc func(task *Task, err error, retried, maxRetry int)
+type ErrorHandlerFunc func(ctx context.Context, task *Task, err error)
 
-// HandleError calls fn(task, err, retried, maxRetry)
-func (fn ErrorHandlerFunc) HandleError(task *Task, err error, retried, maxRetry int) {
-	fn(task, err, retried, maxRetry)
+// HandleError calls fn(ctx, task, err)
+func (fn ErrorHandlerFunc) HandleError(ctx context.Context, task *Task, err error) {
+	fn(ctx, task, err)
 }
 
 // Logger supports logging at various log levels.
@@ -250,7 +260,11 @@ var defaultQueueConfig = map[string]int{
 	base.DefaultQueueName: 1,
 }
 
-const defaultShutdownTimeout = 8 * time.Second
+const (
+	defaultShutdownTimeout = 8 * time.Second
+
+	defaultHealthCheckInterval = 15 * time.Second
+)
 
 // NewServer returns a new Server given a redis connection option
 // and background processing configuration.
@@ -276,6 +290,10 @@ func NewServer(r RedisConnOpt, cfg Config) *Server {
 	if shutdownTimeout == 0 {
 		shutdownTimeout = defaultShutdownTimeout
 	}
+	healthcheckInterval := cfg.HealthCheckInterval
+	if healthcheckInterval == 0 {
+		healthcheckInterval = defaultHealthCheckInterval
+	}
 	logger := log.NewLogger(cfg.Logger)
 	loglevel := cfg.LogLevel
 	if loglevel == level_unspecified {
@@ -283,15 +301,11 @@ func NewServer(r RedisConnOpt, cfg Config) *Server {
 	}
 	logger.SetLevel(toInternalLogLevel(loglevel))
 
-	host, err := os.Hostname()
-	if err != nil {
-		host = "unknown-host"
-	}
-	pid := os.Getpid()
-
 	rdb := rdb.NewRDB(createRedisClient(r))
-	ss := base.NewServerState(host, pid, n, queues, cfg.StrictPriority)
+	starting := make(chan *base.TaskMessage)
+	finished := make(chan *base.TaskMessage)
 	syncCh := make(chan *syncRequest)
+	status := base.NewServerStatus(base.StatusIdle)
 	cancels := base.NewCancelations()
 
 	syncer := newSyncer(syncerParams{
@@ -300,16 +314,20 @@ func NewServer(r RedisConnOpt, cfg Config) *Server {
 		interval:   5 * time.Second,
 	})
 	heartbeater := newHeartbeater(heartbeaterParams{
-		logger:      logger,
-		broker:      rdb,
-		serverState: ss,
-		interval:    5 * time.Second,
+		logger:         logger,
+		broker:         rdb,
+		interval:       5 * time.Second,
+		concurrency:    n,
+		queues:         queues,
+		strictPriority: cfg.StrictPriority,
+		status:         status,
+		starting:       starting,
+		finished:       finished,
 	})
 	scheduler := newScheduler(schedulerParams{
 		logger:   logger,
 		broker:   rdb,
 		interval: 5 * time.Second,
-		queues:   queues,
 	})
 	subscriber := newSubscriber(subscriberParams{
 		logger:       logger,
@@ -319,22 +337,40 @@ func NewServer(r RedisConnOpt, cfg Config) *Server {
 	processor := newProcessor(processorParams{
 		logger:          logger,
 		broker:          rdb,
-		ss:              ss,
 		retryDelayFunc:  delayFunc,
 		syncCh:          syncCh,
 		cancelations:    cancels,
+		concurrency:     n,
+		queues:          queues,
+		strictPriority:  cfg.StrictPriority,
 		errHandler:      cfg.ErrorHandler,
 		shutdownTimeout: shutdownTimeout,
+		starting:        starting,
+		finished:        finished,
+	})
+	recoverer := newRecoverer(recovererParams{
+		logger:         logger,
+		broker:         rdb,
+		retryDelayFunc: delayFunc,
+		interval:       1 * time.Minute,
+	})
+	healthchecker := newHealthChecker(healthcheckerParams{
+		logger:          logger,
+		broker:          rdb,
+		interval:        healthcheckInterval,
+		healthcheckFunc: cfg.HealthCheckFunc,
 	})
 	return &Server{
-		ss:          ss,
-		logger:      logger,
-		broker:      rdb,
-		scheduler:   scheduler,
-		processor:   processor,
-		syncer:      syncer,
-		heartbeater: heartbeater,
-		subscriber:  subscriber,
+		logger:        logger,
+		broker:        rdb,
+		status:        status,
+		scheduler:     scheduler,
+		processor:     processor,
+		syncer:        syncer,
+		heartbeater:   heartbeater,
+		subscriber:    subscriber,
+		recoverer:     recoverer,
+		healthchecker: healthchecker,
 	}
 }
 
@@ -390,20 +426,22 @@ func (srv *Server) Start(handler Handler) error {
 	if handler == nil {
 		return fmt.Errorf("asynq: server cannot run with nil handler")
 	}
-	switch srv.ss.Status() {
+	switch srv.status.Get() {
 	case base.StatusRunning:
 		return fmt.Errorf("asynq: the server is already running")
 	case base.StatusStopped:
 		return ErrServerStopped
 	}
-	srv.ss.SetStatus(base.StatusRunning)
+	srv.status.Set(base.StatusRunning)
 	srv.processor.handler = handler
 
 	srv.logger.Info("Starting processing")
 
 	srv.heartbeater.start(&srv.wg)
+	srv.healthchecker.start(&srv.wg)
 	srv.subscriber.start(&srv.wg)
 	srv.syncer.start(&srv.wg)
+	srv.recoverer.start(&srv.wg)
 	srv.scheduler.start(&srv.wg)
 	srv.processor.start(&srv.wg)
 	return nil
@@ -414,7 +452,7 @@ func (srv *Server) Start(handler Handler) error {
 // active workers to finish processing tasks for duration specified in Config.ShutdownTimeout.
 // If worker didn't finish processing a task during the timeout, the task will be pushed back to Redis.
 func (srv *Server) Stop() {
-	switch srv.ss.Status() {
+	switch srv.status.Get() {
 	case base.StatusIdle, base.StatusStopped:
 		// server is not running, do nothing and return.
 		return
@@ -424,16 +462,19 @@ func (srv *Server) Stop() {
 	// Note: The order of termination is important.
 	// Sender goroutines should be terminated before the receiver goroutines.
 	// processor -> syncer (via syncCh)
+	// processor -> heartbeater (via starting, finished channels)
 	srv.scheduler.terminate()
 	srv.processor.terminate()
+	srv.recoverer.terminate()
 	srv.syncer.terminate()
 	srv.subscriber.terminate()
+	srv.healthchecker.terminate()
 	srv.heartbeater.terminate()
 
 	srv.wg.Wait()
 
 	srv.broker.Close()
-	srv.ss.SetStatus(base.StatusStopped)
+	srv.status.Set(base.StatusStopped)
 
 	srv.logger.Info("Exiting")
 }
@@ -443,6 +484,6 @@ func (srv *Server) Stop() {
 func (srv *Server) Quiet() {
 	srv.logger.Info("Stopping processor")
 	srv.processor.stop()
-	srv.ss.SetStatus(base.StatusQuiet)
+	srv.status.Set(base.StatusQuiet)
 	srv.logger.Info("Processor stopped")
 }

server_test.go

@@ -41,12 +41,12 @@ func TestServer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = c.Enqueue(NewTask("send_email", map[string]interface{}{"recipient_id": 123}))
+	_, err = c.Enqueue(NewTask("send_email", map[string]interface{}{"recipient_id": 123}))
 	if err != nil {
 		t.Errorf("could not enqueue a task: %v", err)
 	}
 
-	err = c.EnqueueAt(time.Now().Add(time.Hour), NewTask("send_email", map[string]interface{}{"recipient_id": 456}))
+	_, err = c.EnqueueAt(time.Now().Add(time.Hour), NewTask("send_email", map[string]interface{}{"recipient_id": 456}))
 	if err != nil {
 		t.Errorf("could not enqueue a task: %v", err)
 	}
@@ -183,15 +183,15 @@ func TestServerWithFlakyBroker(t *testing.T) {
 	}
 
 	for i := 0; i < 10; i++ {
-		err := c.Enqueue(NewTask("enqueued", nil), MaxRetry(i))
+		_, err := c.Enqueue(NewTask("enqueued", nil), MaxRetry(i))
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = c.Enqueue(NewTask("bad_task", nil))
+		_, err = c.Enqueue(NewTask("bad_task", nil))
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = c.EnqueueIn(time.Duration(i)*time.Second, NewTask("scheduled", nil))
+		_, err = c.EnqueueIn(time.Duration(i)*time.Second, NewTask("scheduled", nil))
 		if err != nil {
 			t.Fatal(err)
 		}

syncer.go

@@ -26,8 +26,9 @@ type syncer struct {
 }
 
 type syncRequest struct {
-	fn     func() error // sync operation
-	errMsg string       // error message
+	fn       func() error // sync operation
+	errMsg   string       // error message
+	deadline time.Time    // request should be dropped if deadline has been exceeded
 }
 
 type syncerParams struct {
@@ -72,6 +73,9 @@ func (s *syncer) start(wg *sync.WaitGroup) {
 			case <-time.After(s.interval):
 				var temp []*syncRequest
 				for _, req := range requests {
+					if req.deadline.Before(time.Now()) {
+						continue // drop stale request
+					}
 					if err := req.fn(); err != nil {
 						temp = append(temp, req)
 					}

syncer_test.go

@@ -42,6 +42,7 @@ func TestSyncer(t *testing.T) {
 			fn: func() error {
 				return rdbClient.Done(m)
 			},
+			deadline: time.Now().Add(5 * time.Minute),
 		}
 	}
 
@@ -85,8 +86,9 @@ func TestSyncerRetry(t *testing.T) {
 	}
 
 	syncRequestCh <- &syncRequest{
-		fn:     requestFunc,
-		errMsg: "error",
+		fn:       requestFunc,
+		errMsg:   "error",
+		deadline: time.Now().Add(5 * time.Minute),
 	}
 
 	// allow syncer to retry
@@ -98,3 +100,41 @@ func TestSyncerRetry(t *testing.T) {
 	}
 	mu.Unlock()
 }
+
+func TestSyncerDropsStaleRequests(t *testing.T) {
+	const interval = time.Second
+	syncRequestCh := make(chan *syncRequest)
+	syncer := newSyncer(syncerParams{
+		logger:     testLogger,
+		requestsCh: syncRequestCh,
+		interval:   interval,
+	})
+	var wg sync.WaitGroup
+	syncer.start(&wg)
+
+	var (
+		mu sync.Mutex
+		n  int // number of times request has been processed
+	)
+
+	for i := 0; i < 10; i++ {
+		syncRequestCh <- &syncRequest{
+			fn: func() error {
+				mu.Lock()
+				n++
+				mu.Unlock()
+				return nil
+			},
+			deadline: time.Now().Add(time.Duration(-i) * time.Second), // already exceeded deadline
+		}
+	}
+
+	time.Sleep(2 * interval) // ensure that syncer runs at least once
+	syncer.terminate()
+
+	mu.Lock()
+	if n != 0 {
+		t.Errorf("requests has been processed %d times, want 0", n)
+	}
+	mu.Unlock()
+}

tools/asynq/README.md

@@ -14,6 +14,7 @@ Asynq CLI is a command line tool to monitor the tasks managed by `asynq` package
   - [Delete](#delete)
   - [Kill](#kill)
   - [Cancel](#cancel)
+  - [Pause](#pause)
 - [Config File](#config-file)
 
 ## Installation
@@ -144,6 +145,17 @@ Example:
 
     asynq cancel bnogo8gt6toe23vhef0g
 
+### Pause
+
+Command `pause` pauses the spcified queue. Tasks in paused queues are not processed by servers.
+To resume processing from the queue, use `unpause` command.
+To see which queues are currently paused, use `stats` command.
+
+Example:
+
+    asynq pause email
+    asynq unpause email
+
 ## Config File
 
 You can use a config file to set default values for the flags.

tools/asynq/cmd/del.go

@@ -8,15 +8,14 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/go-redis/redis/v7"
-	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/hibiken/asynq"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 // delCmd represents the del command
 var delCmd = &cobra.Command{
-	Use:   "del [task id]",
+	Use:   "del [task key]",
 	Short: "Deletes a task given an identifier",
 	Long: `Del (asynq del) will delete a task given an identifier.
 
@@ -44,27 +43,12 @@ func init() {
 }
 
 func del(cmd *cobra.Command, args []string) {
-	id, score, qtype, err := parseQueryID(args[0])
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	r := rdb.NewRDB(redis.NewClient(&redis.Options{
+	i := asynq.NewInspector(asynq.RedisClientOpt{
 		Addr:     viper.GetString("uri"),
 		DB:       viper.GetInt("db"),
 		Password: viper.GetString("password"),
-	}))
-	switch qtype {
-	case "s":
-		err = r.DeleteScheduledTask(id, score)
-	case "r":
-		err = r.DeleteRetryTask(id, score)
-	case "d":
-		err = r.DeleteDeadTask(id, score)
-	default:
-		fmt.Println("invalid argument")
-		os.Exit(1)
-	}
+	})
+	err := i.DeleteTaskByKey(args[0])
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)

tools/asynq/cmd/delall.go

@@ -8,8 +8,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/go-redis/redis/v7"
-	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/hibiken/asynq"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -45,20 +44,22 @@ func init() {
 }
 
 func delall(cmd *cobra.Command, args []string) {
-	c := redis.NewClient(&redis.Options{
+	i := asynq.NewInspector(asynq.RedisClientOpt{
 		Addr:     viper.GetString("uri"),
 		DB:       viper.GetInt("db"),
 		Password: viper.GetString("password"),
 	})
-	r := rdb.NewRDB(c)
-	var err error
+	var (
+		n   int
+		err error
+	)
 	switch args[0] {
 	case "scheduled":
-		err = r.DeleteAllScheduledTasks()
+		n, err = i.DeleteAllScheduledTasks()
 	case "retry":
-		err = r.DeleteAllRetryTasks()
+		n, err = i.DeleteAllRetryTasks()
 	case "dead":
-		err = r.DeleteAllDeadTasks()
+		n, err = i.DeleteAllDeadTasks()
 	default:
 		fmt.Printf("error: `asynq delall [state]` only accepts %v as the argument.\n", delallValidArgs)
 		os.Exit(1)
@@ -67,5 +68,5 @@ func delall(cmd *cobra.Command, args []string) {
 		fmt.Println(err)
 		os.Exit(1)
 	}
-	fmt.Printf("Deleted all tasks in %q state\n", args[0])
+	fmt.Printf("Deleted all %d tasks in %q state\n", n, args[0])
 }

tools/asynq/cmd/enq.go

@@ -8,15 +8,14 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/go-redis/redis/v7"
-	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/hibiken/asynq"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 // enqCmd represents the enq command
 var enqCmd = &cobra.Command{
-	Use:   "enq [task id]",
+	Use:   "enq [task key]",
 	Short: "Enqueues a task given an identifier",
 	Long: `Enq (asynq enq) will enqueue a task given an identifier.
 
@@ -47,27 +46,12 @@ func init() {
 }
 
 func enq(cmd *cobra.Command, args []string) {
-	id, score, qtype, err := parseQueryID(args[0])
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	r := rdb.NewRDB(redis.NewClient(&redis.Options{
+	i := asynq.NewInspector(asynq.RedisClientOpt{
 		Addr:     viper.GetString("uri"),
 		DB:       viper.GetInt("db"),
 		Password: viper.GetString("password"),
-	}))
-	switch qtype {
-	case "s":
-		err = r.EnqueueScheduledTask(id, score)
-	case "r":
-		err = r.EnqueueRetryTask(id, score)
-	case "d":
-		err = r.EnqueueDeadTask(id, score)
-	default:
-		fmt.Println("invalid argument")
-		os.Exit(1)
-	}
+	})
+	err := i.EnqueueTaskByKey(args[0])
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)

tools/asynq/cmd/enqall.go

@@ -8,8 +8,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/go-redis/redis/v7"
-	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/hibiken/asynq"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -48,21 +47,22 @@ func init() {
 }
 
 func enqall(cmd *cobra.Command, args []string) {
-	c := redis.NewClient(&redis.Options{
+	i := asynq.NewInspector(asynq.RedisClientOpt{
 		Addr:     viper.GetString("uri"),
 		DB:       viper.GetInt("db"),
 		Password: viper.GetString("password"),
 	})
-	r := rdb.NewRDB(c)
-	var n int64
-	var err error
+	var (
+		n   int
+		err error
+	)
 	switch args[0] {
 	case "scheduled":
-		n, err = r.EnqueueAllScheduledTasks()
+		n, err = i.EnqueueAllScheduledTasks()
 	case "retry":
-		n, err = r.EnqueueAllRetryTasks()
+		n, err = i.EnqueueAllRetryTasks()
 	case "dead":
-		n, err = r.EnqueueAllDeadTasks()
+		n, err = i.EnqueueAllDeadTasks()
 	default:
 		fmt.Printf("error: `asynq enqall [state]` only accepts %v as the argument.\n", enqallValidArgs)
 		os.Exit(1)

tools/asynq/cmd/history.go

@@ -10,8 +10,7 @@ import (
 	"strings"
 	"text/tabwriter"
 
-	"github.com/go-redis/redis/v7"
-	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/hibiken/asynq"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -38,14 +37,13 @@ func init() {
 }
 
 func history(cmd *cobra.Command, args []string) {
-	c := redis.NewClient(&redis.Options{
+	i := asynq.NewInspector(asynq.RedisClientOpt{
 		Addr:     viper.GetString("uri"),
 		DB:       viper.GetInt("db"),
 		Password: viper.GetString("password"),
 	})
-	r := rdb.NewRDB(c)
 
-	stats, err := r.HistoricalStats(days)
+	stats, err := i.History(days)
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -53,7 +51,7 @@ func history(cmd *cobra.Command, args []string) {
 	printDailyStats(stats)
 }
 
-func printDailyStats(stats []*rdb.DailyStats) {
+func printDailyStats(stats []*asynq.DailyStats) {
 	format := strings.Repeat("%v\t", 4) + "\n"
 	tw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0)
 	fmt.Fprintf(tw, format, "Date (UTC)", "Processed", "Failed", "Error Rate")
@@ -65,7 +63,7 @@ func printDailyStats(stats []*rdb.DailyStats) {
 		} else {
 			errrate = fmt.Sprintf("%.2f%%", float64(s.Failed)/float64(s.Processed)*100)
 		}
-		fmt.Fprintf(tw, format, s.Time.Format("2006-01-02"), s.Processed, s.Failed, errrate)
+		fmt.Fprintf(tw, format, s.Date.Format("2006-01-02"), s.Processed, s.Failed, errrate)
 	}
 	tw.Flush()
 }

tools/asynq/cmd/kill.go

@@ -8,15 +8,14 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/go-redis/redis/v7"
-	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/hibiken/asynq"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 // killCmd represents the kill command
 var killCmd = &cobra.Command{
-	Use:   "kill [task id]",
+	Use:   "kill [task key]",
 	Short: "Kills a task given an identifier",
 	Long: `Kill (asynq kill) will put a task in dead state given an identifier.
 
@@ -44,25 +43,12 @@ func init() {
 }
 
 func kill(cmd *cobra.Command, args []string) {
-	id, score, qtype, err := parseQueryID(args[0])
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	r := rdb.NewRDB(redis.NewClient(&redis.Options{
+	i := asynq.NewInspector(asynq.RedisClientOpt{
 		Addr:     viper.GetString("uri"),
 		DB:       viper.GetInt("db"),
 		Password: viper.GetString("password"),
-	}))
-	switch qtype {
-	case "s":
-		err = r.KillScheduledTask(id, score)
-	case "r":
-		err = r.KillRetryTask(id, score)
-	default:
-		fmt.Println("invalid argument")
-		os.Exit(1)
-	}
+	})
+	err := i.KillTaskByKey(args[0])
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)

tools/asynq/cmd/killall.go

@@ -8,8 +8,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/go-redis/redis/v7"
-	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/hibiken/asynq"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -45,19 +44,20 @@ func init() {
 }
 
 func killall(cmd *cobra.Command, args []string) {
-	c := redis.NewClient(&redis.Options{
+	i := asynq.NewInspector(asynq.RedisClientOpt{
 		Addr:     viper.GetString("uri"),
 		DB:       viper.GetInt("db"),
 		Password: viper.GetString("password"),
 	})
-	r := rdb.NewRDB(c)
-	var n int64
-	var err error
+	var (
+		n   int
+		err error
+	)
 	switch args[0] {
 	case "scheduled":
-		n, err = r.KillAllScheduledTasks()
+		n, err = i.KillAllScheduledTasks()
 	case "retry":
-		n, err = r.KillAllRetryTasks()
+		n, err = i.KillAllRetryTasks()
 	default:
 		fmt.Printf("error: `asynq killall [state]` only accepts %v as the argument.\n", killallValidArgs)
 		os.Exit(1)

tools/asynq/cmd/ls.go

@@ -8,13 +8,10 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"strconv"
 	"strings"
 	"time"
 
-	"github.com/go-redis/redis/v7"
-	"github.com/hibiken/asynq/internal/rdb"
-	"github.com/rs/xid"
+	"github.com/hibiken/asynq"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -62,12 +59,11 @@ func ls(cmd *cobra.Command, args []string) {
 		fmt.Println("page number cannot be negative.")
 		os.Exit(1)
 	}
-	c := redis.NewClient(&redis.Options{
+	i := asynq.NewInspector(asynq.RedisClientOpt{
 		Addr:     viper.GetString("uri"),
 		DB:       viper.GetInt("db"),
 		Password: viper.GetString("password"),
 	})
-	r := rdb.NewRDB(c)
 	parts := strings.Split(args[0], ":")
 	switch parts[0] {
 	case "enqueued":
@@ -75,54 +71,23 @@ func ls(cmd *cobra.Command, args []string) {
 			fmt.Printf("error: Missing queue name\n`asynq ls enqueued:[queue name]`\n")
 			os.Exit(1)
 		}
-		listEnqueued(r, parts[1])
+		listEnqueued(i, parts[1])
 	case "inprogress":
-		listInProgress(r)
+		listInProgress(i)
 	case "scheduled":
-		listScheduled(r)
+		listScheduled(i)
 	case "retry":
-		listRetry(r)
+		listRetry(i)
 	case "dead":
-		listDead(r)
+		listDead(i)
 	default:
 		fmt.Printf("error: `asynq ls [state]`\nonly accepts %v as the argument.\n", lsValidArgs)
 		os.Exit(1)
 	}
 }
 
-// queryID returns an identifier used for "enq" command.
-// score is the zset score and queryType should be one
-// of "s", "r" or "d" (scheduled, retry, dead respectively).
-func queryID(id xid.ID, score int64, qtype string) string {
-	const format = "%v:%v:%v"
-	return fmt.Sprintf(format, qtype, score, id)
-}
-
-// parseQueryID is a reverse operation of queryID function.
-// It takes a queryID and return each part of id with proper
-// type if valid, otherwise it reports an error.
-func parseQueryID(queryID string) (id xid.ID, score int64, qtype string, err error) {
-	parts := strings.Split(queryID, ":")
-	if len(parts) != 3 {
-		return xid.NilID(), 0, "", fmt.Errorf("invalid id")
-	}
-	id, err = xid.FromString(parts[2])
-	if err != nil {
-		return xid.NilID(), 0, "", fmt.Errorf("invalid id")
-	}
-	score, err = strconv.ParseInt(parts[1], 10, 64)
-	if err != nil {
-		return xid.NilID(), 0, "", fmt.Errorf("invalid id")
-	}
-	qtype = parts[0]
-	if len(qtype) != 1 || !strings.Contains("srd", qtype) {
-		return xid.NilID(), 0, "", fmt.Errorf("invalid id")
-	}
-	return id, score, qtype, nil
-}
-
-func listEnqueued(r *rdb.RDB, qname string) {
-	tasks, err := r.ListEnqueued(qname, rdb.Pagination{Size: pageSize, Page: pageNum})
+func listEnqueued(i *asynq.Inspector, qname string) {
+	tasks, err := i.ListEnqueuedTasks(qname, asynq.PageSize(pageSize), asynq.Page(pageNum))
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -132,17 +97,16 @@ func listEnqueued(r *rdb.RDB, qname string) {
 		return
 	}
 	cols := []string{"ID", "Type", "Payload", "Queue"}
-	printRows := func(w io.Writer, tmpl string) {
+	printTable(cols, func(w io.Writer, tmpl string) {
 		for _, t := range tasks {
 			fmt.Fprintf(w, tmpl, t.ID, t.Type, t.Payload, t.Queue)
 		}
-	}
-	printTable(cols, printRows)
+	})
 	fmt.Printf("\nShowing %d tasks from page %d\n", len(tasks), pageNum)
 }
 
-func listInProgress(r *rdb.RDB) {
-	tasks, err := r.ListInProgress(rdb.Pagination{Size: pageSize, Page: pageNum})
+func listInProgress(i *asynq.Inspector) {
+	tasks, err := i.ListInProgressTasks(asynq.PageSize(pageSize), asynq.Page(pageNum))
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -152,17 +116,16 @@ func listInProgress(r *rdb.RDB) {
 		return
 	}
 	cols := []string{"ID", "Type", "Payload"}
-	printRows := func(w io.Writer, tmpl string) {
+	printTable(cols, func(w io.Writer, tmpl string) {
 		for _, t := range tasks {
 			fmt.Fprintf(w, tmpl, t.ID, t.Type, t.Payload)
 		}
-	}
-	printTable(cols, printRows)
+	})
 	fmt.Printf("\nShowing %d tasks from page %d\n", len(tasks), pageNum)
 }
 
-func listScheduled(r *rdb.RDB) {
-	tasks, err := r.ListScheduled(rdb.Pagination{Size: pageSize, Page: pageNum})
+func listScheduled(i *asynq.Inspector) {
+	tasks, err := i.ListScheduledTasks(asynq.PageSize(pageSize), asynq.Page(pageNum))
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -171,19 +134,19 @@ func listScheduled(r *rdb.RDB) {
 		fmt.Println("No scheduled tasks")
 		return
 	}
-	cols := []string{"ID", "Type", "Payload", "Process In", "Queue"}
-	printRows := func(w io.Writer, tmpl string) {
+	cols := []string{"Key", "Type", "Payload", "Process In", "Queue"}
+	printTable(cols, func(w io.Writer, tmpl string) {
 		for _, t := range tasks {
-			processIn := fmt.Sprintf("%.0f seconds", t.ProcessAt.Sub(time.Now()).Seconds())
-			fmt.Fprintf(w, tmpl, queryID(t.ID, t.Score, "s"), t.Type, t.Payload, processIn, t.Queue)
+			processIn := fmt.Sprintf("%.0f seconds",
+				t.NextEnqueueAt.Sub(time.Now()).Seconds())
+			fmt.Fprintf(w, tmpl, t.Key(), t.Type, t.Payload, processIn, t.Queue)
 		}
-	}
-	printTable(cols, printRows)
+	})
 	fmt.Printf("\nShowing %d tasks from page %d\n", len(tasks), pageNum)
 }
 
-func listRetry(r *rdb.RDB) {
-	tasks, err := r.ListRetry(rdb.Pagination{Size: pageSize, Page: pageNum})
+func listRetry(i *asynq.Inspector) {
+	tasks, err := i.ListRetryTasks(asynq.PageSize(pageSize), asynq.Page(pageNum))
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -192,24 +155,23 @@ func listRetry(r *rdb.RDB) {
 		fmt.Println("No retry tasks")
 		return
 	}
-	cols := []string{"ID", "Type", "Payload", "Next Retry", "Last Error", "Retried", "Max Retry", "Queue"}
-	printRows := func(w io.Writer, tmpl string) {
+	cols := []string{"Key", "Type", "Payload", "Next Retry", "Last Error", "Retried", "Max Retry", "Queue"}
+	printTable(cols, func(w io.Writer, tmpl string) {
 		for _, t := range tasks {
 			var nextRetry string
-			if d := t.ProcessAt.Sub(time.Now()); d > 0 {
+			if d := t.NextEnqueueAt.Sub(time.Now()); d > 0 {
 				nextRetry = fmt.Sprintf("in %v", d.Round(time.Second))
 			} else {
 				nextRetry = "right now"
 			}
-			fmt.Fprintf(w, tmpl, queryID(t.ID, t.Score, "r"), t.Type, t.Payload, nextRetry, t.ErrorMsg, t.Retried, t.Retry, t.Queue)
+			fmt.Fprintf(w, tmpl, t.Key(), t.Type, t.Payload, nextRetry, t.ErrorMsg, t.Retried, t.MaxRetry, t.Queue)
 		}
-	}
-	printTable(cols, printRows)
+	})
 	fmt.Printf("\nShowing %d tasks from page %d\n", len(tasks), pageNum)
 }
 
-func listDead(r *rdb.RDB) {
-	tasks, err := r.ListDead(rdb.Pagination{Size: pageSize, Page: pageNum})
+func listDead(i *asynq.Inspector) {
+	tasks, err := i.ListDeadTasks(asynq.PageSize(pageSize), asynq.Page(pageNum))
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -218,12 +180,11 @@ func listDead(r *rdb.RDB) {
 		fmt.Println("No dead tasks")
 		return
 	}
-	cols := []string{"ID", "Type", "Payload", "Last Failed", "Last Error", "Queue"}
-	printRows := func(w io.Writer, tmpl string) {
+	cols := []string{"Key", "Type", "Payload", "Last Failed", "Last Error", "Queue"}
+	printTable(cols, func(w io.Writer, tmpl string) {
 		for _, t := range tasks {
-			fmt.Fprintf(w, tmpl, queryID(t.ID, t.Score, "d"), t.Type, t.Payload, t.LastFailedAt, t.ErrorMsg, t.Queue)
+			fmt.Fprintf(w, tmpl, t.Key(), t.Type, t.Payload, t.LastFailedAt, t.ErrorMsg, t.Queue)
 		}
-	}
-	printTable(cols, printRows)
+	})
 	fmt.Printf("\nShowing %d tasks from page %d\n", len(tasks), pageNum)
 }

tools/asynq/cmd/migrate.go

@@ -0,0 +1,212 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/go-redis/redis/v7"
+	"github.com/google/uuid"
+	"github.com/hibiken/asynq/internal/base"
+	"github.com/spf13/cast"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// migrateCmd represents the migrate command
+var migrateCmd = &cobra.Command{
+	Use:   "migrate",
+	Short: fmt.Sprintf("Migrate all tasks to be compatible with asynq@%s", base.Version),
+	Long:  fmt.Sprintf("Migrate (asynq migrate) will convert all tasks in redis to be compatible with asynq@%s.", base.Version),
+	Run:   migrate,
+}
+
+func init() {
+	rootCmd.AddCommand(migrateCmd)
+}
+
+func migrate(cmd *cobra.Command, args []string) {
+	c := redis.NewClient(&redis.Options{
+		Addr:     viper.GetString("uri"),
+		DB:       viper.GetInt("db"),
+		Password: viper.GetString("password"),
+	})
+
+	lists := []string{base.InProgressQueue}
+	allQueues, err := c.SMembers(base.AllQueues).Result()
+	if err != nil {
+		fmt.Printf("error: could not read all queues: %v", err)
+		os.Exit(1)
+	}
+	lists = append(lists, allQueues...)
+	for _, key := range lists {
+		if err := migrateList(c, key); err != nil {
+			fmt.Printf("error: %v", err)
+			os.Exit(1)
+		}
+	}
+
+	zsets := []string{base.ScheduledQueue, base.RetryQueue, base.DeadQueue}
+	for _, key := range zsets {
+		if err := migrateZSet(c, key); err != nil {
+			fmt.Printf("error: %v", err)
+			os.Exit(1)
+		}
+	}
+}
+
+type oldTaskMessage struct {
+	// Unchanged
+	Type      string
+	Payload   map[string]interface{}
+	ID        uuid.UUID
+	Queue     string
+	Retry     int
+	Retried   int
+	ErrorMsg  string
+	UniqueKey string
+
+	// Following fields have changed.
+
+	// Deadline specifies the deadline for the task.
+	// Task won't be processed if it exceeded its deadline.
+	// The string shoulbe be in RFC3339 format.
+	//
+	// time.Time's zero value means no deadline.
+	Timeout string
+
+	// Deadline specifies the deadline for the task.
+	// Task won't be processed if it exceeded its deadline.
+	// The string shoulbe be in RFC3339 format.
+	//
+	// time.Time's zero value means no deadline.
+	Deadline string
+}
+
+var defaultTimeout = 30 * time.Minute
+
+func convertMessage(old *oldTaskMessage) (*base.TaskMessage, error) {
+	timeout, err := time.ParseDuration(old.Timeout)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse Timeout field of %+v", old)
+	}
+	deadline, err := time.Parse(time.RFC3339, old.Deadline)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse Deadline field of %+v", old)
+	}
+	if timeout == 0 && deadline.IsZero() {
+		timeout = defaultTimeout
+	}
+	if deadline.IsZero() {
+		// Zero value used to be time.Time{},
+		// in the new schema zero value is represented by
+		// zero in Unix time.
+		deadline = time.Unix(0, 0)
+	}
+	return &base.TaskMessage{
+		Type:      old.Type,
+		Payload:   old.Payload,
+		ID:        uuid.New(),
+		Queue:     old.Queue,
+		Retry:     old.Retry,
+		Retried:   old.Retried,
+		ErrorMsg:  old.ErrorMsg,
+		UniqueKey: old.UniqueKey,
+		Timeout:   int64(timeout.Seconds()),
+		Deadline:  deadline.Unix(),
+	}, nil
+}
+
+func deserialize(s string) (*base.TaskMessage, error) {
+	// Try deserializing as old message.
+	d := json.NewDecoder(strings.NewReader(s))
+	d.UseNumber()
+	var old *oldTaskMessage
+	if err := d.Decode(&old); err != nil {
+		// Try deserializing as new message.
+		d = json.NewDecoder(strings.NewReader(s))
+		d.UseNumber()
+		var msg *base.TaskMessage
+		if err := d.Decode(&msg); err != nil {
+			return nil, fmt.Errorf("could not deserialize %s into task message: %v", s, err)
+		}
+		return msg, nil
+	}
+	return convertMessage(old)
+}
+
+func migrateZSet(c *redis.Client, key string) error {
+	if c.Exists(key).Val() == 0 {
+		// skip if key doesn't exist.
+		return nil
+	}
+	res, err := c.ZRangeWithScores(key, 0, -1).Result()
+	if err != nil {
+		return err
+	}
+	var msgs []*redis.Z
+	for _, z := range res {
+		s, err := cast.ToStringE(z.Member)
+		if err != nil {
+			return fmt.Errorf("could not cast to string: %v", err)
+		}
+		msg, err := deserialize(s)
+		if err != nil {
+			return err
+		}
+		encoded, err := base.EncodeMessage(msg)
+		if err != nil {
+			return fmt.Errorf("could not encode message from %q: %v", key, err)
+		}
+		msgs = append(msgs, &redis.Z{Score: z.Score, Member: encoded})
+	}
+	if err := c.Rename(key, key+":backup").Err(); err != nil {
+		return fmt.Errorf("could not rename key %q: %v", key, err)
+	}
+	if err := c.ZAdd(key, msgs...).Err(); err != nil {
+		return fmt.Errorf("could not write new messages to %q: %v", key, err)
+	}
+	if err := c.Del(key + ":backup").Err(); err != nil {
+		return fmt.Errorf("could not delete back up key %q: %v", key+":backup", err)
+	}
+	return nil
+}
+
+func migrateList(c *redis.Client, key string) error {
+	if c.Exists(key).Val() == 0 {
+		// skip if key doesn't exist.
+		return nil
+	}
+	res, err := c.LRange(key, 0, -1).Result()
+	if err != nil {
+		return err
+	}
+	var msgs []interface{}
+	for _, s := range res {
+		msg, err := deserialize(s)
+		if err != nil {
+			return err
+		}
+		encoded, err := base.EncodeMessage(msg)
+		if err != nil {
+			return fmt.Errorf("could not encode message from %q: %v", key, err)
+		}
+		msgs = append(msgs, encoded)
+	}
+	if err := c.Rename(key, key+":backup").Err(); err != nil {
+		return fmt.Errorf("could not rename key %q: %v", key, err)
+	}
+	if err := c.LPush(key, msgs...).Err(); err != nil {
+		return fmt.Errorf("could not write new messages to %q: %v", key, err)
+	}
+	if err := c.Del(key + ":backup").Err(); err != nil {
+		return fmt.Errorf("could not delete back up key %q: %v", key+":backup", err)
+	}
+	return nil
+}

tools/asynq/cmd/pause.go

@@ -0,0 +1,47 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/go-redis/redis/v7"
+	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// pauseCmd represents the pause command
+var pauseCmd = &cobra.Command{
+	Use:   "pause [queue name]",
+	Short: "Pauses the specified queue",
+	Long: `Pause (asynq pause) will pause the specified queue.
+Asynq servers will not process tasks from paused queues.
+Use the "unpause" command to resume a paused queue.
+
+Example: asynq pause default -> Pause the "default" queue`,
+	Args: cobra.ExactValidArgs(1),
+	Run:  pause,
+}
+
+func init() {
+	rootCmd.AddCommand(pauseCmd)
+}
+
+func pause(cmd *cobra.Command, args []string) {
+	c := redis.NewClient(&redis.Options{
+		Addr:     viper.GetString("uri"),
+		DB:       viper.GetInt("db"),
+		Password: viper.GetString("password"),
+	})
+	r := rdb.NewRDB(c)
+	err := r.Pause(args[0])
+	if err != nil {
+		fmt.Printf("error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("Successfully paused queue %q\n", args[0])
+}

tools/asynq/cmd/root.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"text/tabwriter"
 
+	"github.com/hibiken/asynq/internal/base"
 	"github.com/spf13/cobra"
 
 	homedir "github.com/mitchellh/go-homedir"
@@ -26,9 +27,20 @@ var password string
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
-	Use:   "asynq",
-	Short: "A monitoring tool for asynq queues",
-	Long:  `Asynq is a montoring CLI to inspect tasks and queues managed by asynq.`,
+	Use:     "asynq",
+	Short:   "A monitoring tool for asynq queues",
+	Long:    `Asynq is a montoring CLI to inspect tasks and queues managed by asynq.`,
+	Version: base.Version,
+}
+
+var versionOutput = fmt.Sprintf("asynq version %s\n", base.Version)
+
+var versionCmd = &cobra.Command{
+	Use:    "version",
+	Hidden: true,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Print(versionOutput)
+	},
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.
@@ -43,6 +55,9 @@ func Execute() {
 func init() {
 	cobra.OnInitialize(initConfig)
 
+	rootCmd.AddCommand(versionCmd)
+	rootCmd.SetVersionTemplate(versionOutput)
+
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file to set flag defaut values (default is $HOME/.asynq.yaml)")
 	rootCmd.PersistentFlags().StringVarP(&uri, "uri", "u", "127.0.0.1:6379", "redis server URI")
 	rootCmd.PersistentFlags().IntVarP(&db, "db", "n", 0, "redis database number (default is 0)")

tools/asynq/cmd/stats.go

@@ -7,7 +7,6 @@ package cmd
 import (
 	"fmt"
 	"os"
-	"sort"
 	"strconv"
 	"strings"
 	"text/tabwriter"
@@ -96,24 +95,31 @@ func printStates(s *rdb.Stats) {
 	tw.Flush()
 }
 
-func printQueues(queues map[string]int) {
-	var qnames, seps, counts []string
-	for q := range queues {
-		qnames = append(qnames, strings.Title(q))
+func printQueues(queues []*rdb.Queue) {
+	var headers, seps, counts []string
+	for _, q := range queues {
+		title := queueTitle(q)
+		headers = append(headers, title)
+		seps = append(seps, strings.Repeat("-", len(title)))
+		counts = append(counts, strconv.Itoa(q.Size))
 	}
-	sort.Strings(qnames) // sort for stable order
-	for _, q := range qnames {
-		seps = append(seps, strings.Repeat("-", len(q)))
-		counts = append(counts, strconv.Itoa(queues[strings.ToLower(q)]))
-	}
-	format := strings.Repeat("%v\t", len(qnames)) + "\n"
+	format := strings.Repeat("%v\t", len(headers)) + "\n"
 	tw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0)
-	fmt.Fprintf(tw, format, toInterfaceSlice(qnames)...)
+	fmt.Fprintf(tw, format, toInterfaceSlice(headers)...)
 	fmt.Fprintf(tw, format, toInterfaceSlice(seps)...)
 	fmt.Fprintf(tw, format, toInterfaceSlice(counts)...)
 	tw.Flush()
 }
 
+func queueTitle(q *rdb.Queue) string {
+	var b strings.Builder
+	b.WriteString(strings.Title(q.Name))
+	if q.Paused {
+		b.WriteString(" (Paused)")
+	}
+	return b.String()
+}
+
 func printStats(s *rdb.Stats) {
 	format := strings.Repeat("%v\t", 3) + "\n"
 	tw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0)

tools/asynq/cmd/unpause.go

@@ -0,0 +1,46 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/go-redis/redis/v7"
+	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// unpauseCmd represents the unpause command
+var unpauseCmd = &cobra.Command{
+	Use:   "unpause [queue name]",
+	Short: "Unpauses the specified queue",
+	Long: `Unpause (asynq unpause) will unpause the specified queue.
+Asynq servers will process tasks from unpaused/resumed queues.
+
+Example: asynq unpause default -> Resume the "default" queue`,
+	Args: cobra.ExactValidArgs(1),
+	Run:  unpause,
+}
+
+func init() {
+	rootCmd.AddCommand(unpauseCmd)
+}
+
+func unpause(cmd *cobra.Command, args []string) {
+	c := redis.NewClient(&redis.Options{
+		Addr:     viper.GetString("uri"),
+		DB:       viper.GetInt("db"),
+		Password: viper.GetString("password"),
+	})
+	r := rdb.NewRDB(c)
+	err := r.Unpause(args[0])
+	if err != nil {
+		fmt.Printf("error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("Successfully resumed queue %q\n", args[0])
+}

tools/asynq/cmd/workers.go

@@ -61,7 +61,7 @@ func workers(cmd *cobra.Command, args []string) {
 		if x.Started != y.Started {
 			return x.Started.Before(y.Started)
 		}
-		return x.ID.String() < y.ID.String()
+		return x.ID < y.ID
 	})
 
 	cols := []string{"Process", "ID", "Type", "Payload", "Queue", "Started"}

tools/go.mod

@@ -4,9 +4,10 @@ go 1.13
 
 require (
 	github.com/go-redis/redis/v7 v7.2.0
+	github.com/google/uuid v1.1.1
 	github.com/hibiken/asynq v0.4.0
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/rs/xid v1.2.1
+	github.com/spf13/cast v1.3.1
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/viper v1.6.2
 )

tools/go.sum

@@ -1,4 +1,5 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -15,6 +16,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
@@ -24,10 +26,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U=
-github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
-github.com/go-redis/redis/v7 v7.1.0 h1:I4C4a8UGbFejiVjtYVTRVOiMIJ5pm5Yru6ibvDX/OS0=
-github.com/go-redis/redis/v7 v7.1.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-redis/redis/v7 v7.2.0 h1:CrCexy/jYWZjW0AyVoHlcJUeZN19VWlbepTh1Vq6dJs=
 github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -38,10 +36,15 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -49,19 +52,22 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hibiken/asynq v0.4.0 h1:NvAfYX0DRe04WgGMKRg5oX7bs6ktv2fu9YwB6O356FI=
-github.com/hibiken/asynq v0.4.0/go.mod h1:dtrVkxCsGPVhVNHMDXAH7lFq64kbj43+G6lt4FQZfW4=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
@@ -74,15 +80,14 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
-github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
@@ -94,18 +99,16 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
-github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
@@ -113,17 +116,13 @@ github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
-github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
-github.com/spf13/viper v1.6.0/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
 github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E=
 github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
@@ -148,6 +147,7 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -166,12 +166,14 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -180,11 +182,14 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=