v0.9.2

running

.travis.yml

@@ -5,6 +5,7 @@ git:
 go: [1.13.x, 1.14.x]
 script:
   - go test -race -v -coverprofile=coverage.txt -covermode=atomic ./...
+  - go test -run=XXX -bench=. -loglevel=debug ./...
 services:
   - redis-server
 after_success:

.travis/benchcmp.sh

@@ -3,13 +3,16 @@ if [ "${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" != "master" ]; then
     cd ${TRAVIS_BUILD_DIR}/.. && \
     git clone ${REMOTE_URL} "${TRAVIS_REPO_SLUG}-bench" && \
     cd "${TRAVIS_REPO_SLUG}-bench" && \
+
     # Benchmark master
     git checkout master && \
     go test -run=XXX -bench=. ./... > master.txt && \
+
     # Benchmark feature branch
     git checkout ${TRAVIS_COMMIT} && \
     go test -run=XXX -bench=. ./... > feature.txt && \
-    go get -u golang.org/x/tools/cmd/benchcmp && \
+
     # compare two benchmarks
+    go get -u golang.org/x/tools/cmd/benchcmp && \
     benchcmp master.txt feature.txt;
 fi

CHANGELOG.md

@@ -7,6 +7,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.9.2] - 2020-06-08
+
+### Added
+
+- The `pause` and `unpause` commands were added to the CLI. See README for the CLI for details.
+
+## [0.9.1] - 2020-05-29
+
+### Added
+
+- `GetTaskID`, `GetRetryCount`, and `GetMaxRetry` functions were added to extract task metadata from context.
+
+## [0.9.0] - 2020-05-16
+
+### Changed
+
+- `Logger` interface has changed. Please see the godoc for the new interface.
+
+### Added
+
+- `LogLevel` type is added. Server's log level can be specified through `LogLevel` field in `Config`.
+
+## [0.8.3] - 2020-05-08
+
+### Added
+
+- `Close` method is added to `Client`.
+
+## [0.8.2] - 2020-05-03
+
+### Fixed
+
+- [Fixed cancelfunc leak](https://github.com/hibiken/asynq/pull/145)
+
+## [0.8.1] - 2020-04-27
+
+### Added
+
+- `ParseRedisURI` helper function is added to create a `RedisConnOpt` from a URI string.
+- `SetDefaultOptions` method is added to `Client`.
+
 ## [0.8.0] - 2020-04-19
 
 ### Changed

README.md

@@ -15,7 +15,7 @@ Highlevel overview of how Asynq works:
 
 - Client puts task on a queue
 - Server pulls task off queues and starts a worker goroutine for each task
-- Workers process tasks concurrently
+- Tasks are processed concurrently by multiple workers
 
 Task queues are used as a mechanism to distribute work across multiple machines.  
 A system can consist of multiple worker servers and brokers, giving way to high availability and horizontal scaling.
@@ -24,24 +24,25 @@ A system can consist of multiple worker servers and brokers, giving way to high
 
 ## Stability and Compatibility
 
-**Important Note**: Current major version is zero (v0.x.x) to accomodate rapid development and fast iteration while getting early feedback from users. The public API could change without a major version update before v1.0.0 release.
+**Important Note**: Current major version is zero (v0.x.x) to accomodate rapid development and fast iteration while getting early feedback from users (Feedback on APIs are appreciated!). The public API could change without a major version update before v1.0.0 release.
 
 **Status**: The library is currently undergoing heavy development with frequent, breaking API changes.
 
 ## Features
 
-- Guaranteed at least one execution of a task
+- Guaranteed [at least one execution](https://www.cloudcomputingpatterns.org/at_least_once_delivery/) of a task
 - Scheduling of tasks
 - Durability since tasks are written to Redis
-- Retries of failed tasks
-- Concurrency management via configuration
-- Weighted priority queues
-- Strict priority queues
+- [Retries](https://github.com/hibiken/asynq/wiki/Task-Retry) of failed tasks
+- [Weighted priority queues](https://github.com/hibiken/asynq/wiki/Priority-Queues#weighted-priority-queues)
+- [Strict priority queues](https://github.com/hibiken/asynq/wiki/Priority-Queues#strict-priority-queues)
 - Low latency to add a task since writes are fast in Redis
-- De-duplication of tasks using unique option
-- Allow timeout and deadline per task
-- Flexible handler interface with support for middlewares
-- CLI to inspect and remote-control queues and tasks
+- De-duplication of tasks using [unique option](https://github.com/hibiken/asynq/wiki/Unique-Tasks)
+- Allow [timeout and deadline per task](https://github.com/hibiken/asynq/wiki/Task-Timeout-and-Cancelation)
+- [Flexible handler interface with support for middlewares](https://github.com/hibiken/asynq/wiki/Handler-Deep-Dive)
+- [Ability to pause queue](/tools/asynq/README.md#pause) to stop processing tasks from the queue
+- [Support Redis Sentinels](https://github.com/hibiken/asynq/wiki/Automatic-Failover) for HA
+- [CLI](#command-line-tool) to inspect and remote-control queues and tasks
 
 ## Quickstart
 
@@ -62,13 +63,16 @@ import (
     "github.com/hibiken/asynq"
 )
 
-// A list of background task types.
+// A list of task types.
 const (
     EmailDelivery   = "email:deliver"
     ImageProcessing = "image:process"
 )
 
-// Write function NewXXXTask to create a task.
+//----------------------------------------------
+// Write a function NewXXXTask to create a task.
+// A task consists of a type and a payload.
+//----------------------------------------------
 
 func NewEmailDeliveryTask(userID int, tmplID string) *asynq.Task {
     payload := map[string]interface{}{"user_id": userID, "template_id": tmplID}
@@ -80,8 +84,13 @@ func NewImageProcessingTask(src, dst string) *asynq.Task {
     return asynq.NewTask(ImageProcessing, payload)
 }
 
-// Write function HandleXXXTask to handle the given task.
-// NOTE: It satisfies the asynq.HandlerFunc interface.
+//---------------------------------------------------------------
+// Write a function HandleXXXTask to handle the input task.
+// Note that it satisfies the asynq.HandlerFunc interface.
+// 
+// Handler doesn't need to be a function. You can define a type 
+// that satisfies asynq.Handler interface. See examples below.
+//---------------------------------------------------------------
 
 func HandleEmailDeliveryTask(ctx context.Context, t *asynq.Task) error {
     userID, err := t.Payload.GetInt("user_id")
@@ -97,7 +106,12 @@ func HandleEmailDeliveryTask(ctx context.Context, t *asynq.Task) error {
     return nil
 }
 
-func HandleImageProcessingTask(ctx context.Context, t *asynq.Task) error {
+// ImageProcessor implements asynq.Handler interface.
+type ImageProcesser struct {
+    // ... fields for struct
+}
+
+func (p *ImageProcessor) ProcessTask(ctx context.Context, t *asynq.Task) error {
     src, err := t.Payload.GetString("src")
     if err != nil {
         return err
@@ -110,6 +124,10 @@ func HandleImageProcessingTask(ctx context.Context, t *asynq.Task) error {
     // Image processing logic ...
     return nil
 }
+
+func NewImageProcessor() *ImageProcessor {
+    // ... return an instance
+}
 ```
 
 In your web application code, import the above package and use [`Client`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#Client) to put tasks on the queue.  
@@ -131,8 +149,12 @@ const redisAddr = "127.0.0.1:6379"
 func main() {
     r := asynq.RedisClientOpt{Addr: redisAddr}
     c := asynq.NewClient(r)
+    defer c.Close()
 
+    // ------------------------------------------------------
     // Example 1: Enqueue task to be processed immediately.
+    //            Use (*Client).Enqueue method.
+    // ------------------------------------------------------
 
     t := tasks.NewEmailDeliveryTask(42, "some:template:id")
     err := c.Enqueue(t)
@@ -141,7 +163,10 @@ func main() {
     }
 
 
+    // ------------------------------------------------------------
     // Example 2: Schedule task to be processed in the future.
+    //            Use (*Client).EnqueueIn or (*Client).EnqueueAt.
+    // ------------------------------------------------------------
 
     t = tasks.NewEmailDeliveryTask(42, "other:template:id")
     err = c.EnqueueIn(24*time.Hour, t)
@@ -150,18 +175,33 @@ func main() {
     }
 
 
-    // Example 3: Pass options to tune task processing behavior.
+    // ----------------------------------------------------------------------------
+    // Example 3: Set options to tune task processing behavior.
     //            Options include MaxRetry, Queue, Timeout, Deadline, Unique etc.
+    // ----------------------------------------------------------------------------
+
+    c.SetDefaultOptions(tasks.ImageProcessing, asynq.MaxRetry(10), asynq.Timeout(time.Minute))
 
     t = tasks.NewImageProcessingTask("some/blobstore/url", "other/blobstore/url")
-    err = c.Enqueue(t, asynq.MaxRetry(10), asynq.Queue("critical"), asynq.Timeout(time.Minute))
+    err = c.Enqueue(t)
+    if err != nil {
+        log.Fatal("could not enqueue task: %v", err)
+    }
+
+    // ---------------------------------------------------------------------------
+    // Example 4: Pass options to tune task processing behavior at enqueue time.
+    //            Options passed at enqueue time override default ones, if any.
+    // ---------------------------------------------------------------------------
+
+    t = tasks.NewImageProcessingTask("some/blobstore/url", "other/blobstore/url")
+    err = c.Enqueue(t, asynq.Queue("critical"), asynq.Timeout(30*time.Second))
     if err != nil {
         log.Fatal("could not enqueue task: %v", err)
     }
 }
 ```
 
-Next, create a work server binary to process these tasks in the background.  
+Next, create a worker server to process these tasks in the background.  
 To start the background workers, use [`Server`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#Server) and provide your [`Handler`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#Handler) to process the tasks.
 
 You can optionally use [`ServeMux`](https://pkg.go.dev/github.com/hibiken/asynq?tab=doc#ServeMux) to create a handler, just as you would with [`"net/http"`](https://golang.org/pkg/net/http/) Handler.
@@ -170,6 +210,8 @@ You can optionally use [`ServeMux`](https://pkg.go.dev/github.com/hibiken/asynq?
 package main
 
 import (
+    "log"
+
     "github.com/hibiken/asynq"
     "your/app/package/tasks"
 )
@@ -194,7 +236,7 @@ func main() {
     // mux maps a type to a handler
     mux := asynq.NewServeMux()
     mux.HandleFunc(tasks.EmailDelivery, tasks.HandleEmailDeliveryTask)
-    mux.HandleFunc(tasks.ImageProcessing, tasks.HandleImageProcessingTask)
+    mux.Handle(tasks.ImageProcessing, tasks.NewImageProcessor())
     // ...register other handlers...
 
     if err := srv.Run(mux); err != nil {

asynq.go

@@ -7,6 +7,9 @@ package asynq
 import (
 	"crypto/tls"
 	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
 
 	"github.com/go-redis/redis/v7"
 )
@@ -94,6 +97,79 @@ type RedisFailoverClientOpt struct {
 	TLSConfig *tls.Config
 }
 
+// ParseRedisURI parses redis uri string and returns RedisConnOpt if uri is valid.
+// It returns a non-nil error if uri cannot be parsed.
+//
+// Three URI schemes are supported, which are redis:, redis-socket:, and redis-sentinel:.
+// Supported formats are:
+//     redis://[:password@]host[:port][/dbnumber]
+//     redis-socket://[:password@]path[?db=dbnumber]
+//     redis-sentinel://[:password@]host1[:port][,host2:[:port]][,hostN:[:port]][?master=masterName]
+func ParseRedisURI(uri string) (RedisConnOpt, error) {
+	u, err := url.Parse(uri)
+	if err != nil {
+		return nil, fmt.Errorf("asynq: could not parse redis uri: %v", err)
+	}
+	switch u.Scheme {
+	case "redis":
+		return parseRedisURI(u)
+	case "redis-socket":
+		return parseRedisSocketURI(u)
+	case "redis-sentinel":
+		return parseRedisSentinelURI(u)
+	default:
+		return nil, fmt.Errorf("asynq: unsupported uri scheme: %q", u.Scheme)
+	}
+}
+
+func parseRedisURI(u *url.URL) (RedisConnOpt, error) {
+	var db int
+	var err error
+	if len(u.Path) > 0 {
+		xs := strings.Split(strings.Trim(u.Path, "/"), "/")
+		db, err = strconv.Atoi(xs[0])
+		if err != nil {
+			return nil, fmt.Errorf("asynq: could not parse redis uri: database number should be the first segment of the path")
+		}
+	}
+	var password string
+	if v, ok := u.User.Password(); ok {
+		password = v
+	}
+	return RedisClientOpt{Addr: u.Host, DB: db, Password: password}, nil
+}
+
+func parseRedisSocketURI(u *url.URL) (RedisConnOpt, error) {
+	const errPrefix = "asynq: could not parse redis socket uri"
+	if len(u.Path) == 0 {
+		return nil, fmt.Errorf("%s: path does not exist", errPrefix)
+	}
+	q := u.Query()
+	var db int
+	var err error
+	if n := q.Get("db"); n != "" {
+		db, err = strconv.Atoi(n)
+		if err != nil {
+			return nil, fmt.Errorf("%s: query param `db` should be a number", errPrefix)
+		}
+	}
+	var password string
+	if v, ok := u.User.Password(); ok {
+		password = v
+	}
+	return RedisClientOpt{Network: "unix", Addr: u.Path, DB: db, Password: password}, nil
+}
+
+func parseRedisSentinelURI(u *url.URL) (RedisConnOpt, error) {
+	addrs := strings.Split(u.Host, ",")
+	master := u.Query().Get("master")
+	var password string
+	if v, ok := u.User.Password(); ok {
+		password = v
+	}
+	return RedisFailoverClientOpt{MasterName: master, SentinelAddrs: addrs, Password: password}, nil
+}
+
 // createRedisClient returns a redis client given a redis connection configuration.
 //
 // Passing an unexpected type as a RedisConnOpt argument will cause panic.

asynq_test.go

@@ -5,7 +5,7 @@
 package asynq
 
 import (
-	"os"
+	"flag"
 	"sort"
 	"testing"
 
@@ -15,16 +15,28 @@ import (
 	"github.com/hibiken/asynq/internal/log"
 )
 
-// This file defines test helper functions used by
-// other test files.
+//============================================================================
+// This file defines helper functions and variables used in other test files.
+//============================================================================
 
-// redis used for package testing.
-const (
-	redisAddr = "localhost:6379"
-	redisDB   = 14
+// variables used for package testing.
+var (
+	redisAddr string
+	redisDB   int
+
+	testLogLevel = FatalLevel
 )
 
-var testLogger = log.NewLogger(os.Stderr)
+var testLogger *log.Logger
+
+func init() {
+	flag.StringVar(&redisAddr, "redis_addr", "localhost:6379", "redis address to use in testing")
+	flag.IntVar(&redisDB, "redis_db", 14, "redis db number to use in testing")
+	flag.Var(&testLogLevel, "loglevel", "log level to use in testing")
+
+	testLogger = log.NewLogger(nil)
+	testLogger.SetLevel(toInternalLogLevel(testLogLevel))
+}
 
 func setup(tb testing.TB) *redis.Client {
 	tb.Helper()
@@ -44,3 +56,106 @@ var sortTaskOpt = cmp.Transformer("SortMsg", func(in []*Task) []*Task {
 	})
 	return out
 })
+
+func TestParseRedisURI(t *testing.T) {
+	tests := []struct {
+		uri  string
+		want RedisConnOpt
+	}{
+		{
+			"redis://localhost:6379",
+			RedisClientOpt{Addr: "localhost:6379"},
+		},
+		{
+			"redis://localhost:6379/3",
+			RedisClientOpt{Addr: "localhost:6379", DB: 3},
+		},
+		{
+			"redis://:mypassword@localhost:6379",
+			RedisClientOpt{Addr: "localhost:6379", Password: "mypassword"},
+		},
+		{
+			"redis://:mypassword@127.0.0.1:6379/11",
+			RedisClientOpt{Addr: "127.0.0.1:6379", Password: "mypassword", DB: 11},
+		},
+		{
+			"redis-socket:///var/run/redis/redis.sock",
+			RedisClientOpt{Network: "unix", Addr: "/var/run/redis/redis.sock"},
+		},
+		{
+			"redis-socket://:mypassword@/var/run/redis/redis.sock",
+			RedisClientOpt{Network: "unix", Addr: "/var/run/redis/redis.sock", Password: "mypassword"},
+		},
+		{
+			"redis-socket:///var/run/redis/redis.sock?db=7",
+			RedisClientOpt{Network: "unix", Addr: "/var/run/redis/redis.sock", DB: 7},
+		},
+		{
+			"redis-socket://:mypassword@/var/run/redis/redis.sock?db=12",
+			RedisClientOpt{Network: "unix", Addr: "/var/run/redis/redis.sock", Password: "mypassword", DB: 12},
+		},
+		{
+			"redis-sentinel://localhost:5000,localhost:5001,localhost:5002?master=mymaster",
+			RedisFailoverClientOpt{
+				MasterName:    "mymaster",
+				SentinelAddrs: []string{"localhost:5000", "localhost:5001", "localhost:5002"},
+			},
+		},
+		{
+			"redis-sentinel://:mypassword@localhost:5000,localhost:5001,localhost:5002?master=mymaster",
+			RedisFailoverClientOpt{
+				MasterName:    "mymaster",
+				SentinelAddrs: []string{"localhost:5000", "localhost:5001", "localhost:5002"},
+				Password:      "mypassword",
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		got, err := ParseRedisURI(tc.uri)
+		if err != nil {
+			t.Errorf("ParseRedisURI(%q) returned an error: %v", tc.uri, err)
+			continue
+		}
+
+		if diff := cmp.Diff(tc.want, got); diff != "" {
+			t.Errorf("ParseRedisURI(%q) = %+v, want %+v\n(-want,+got)\n%s", tc.uri, got, tc.want, diff)
+		}
+	}
+}
+
+func TestParseRedisURIErrors(t *testing.T) {
+	tests := []struct {
+		desc string
+		uri  string
+	}{
+		{
+			"unsupported scheme",
+			"rdb://localhost:6379",
+		},
+		{
+			"missing scheme",
+			"localhost:6379",
+		},
+		{
+			"multiple db numbers",
+			"redis://localhost:6379/1,2,3",
+		},
+		{
+			"missing path for socket connection",
+			"redis-socket://?db=one",
+		},
+		{
+			"non integer for db numbers for socket",
+			"redis-socket:///some/path/to/redis?db=one",
+		},
+	}
+
+	for _, tc := range tests {
+		_, err := ParseRedisURI(tc.uri)
+		if err == nil {
+			t.Errorf("%s: ParseRedisURI(%q) succeeded for malformed input, want error",
+				tc.desc, tc.uri)
+		}
+	}
+}

benchmark_test.go

@@ -7,7 +7,6 @@ package asynq
 import (
 	"context"
 	"fmt"
-	"math/rand"
 	"sync"
 	"testing"
 	"time"
@@ -29,6 +28,7 @@ func BenchmarkEndToEndSimple(b *testing.B) {
 			RetryDelayFunc: func(n int, err error, t *Task) time.Duration {
 				return time.Second
 			},
+			LogLevel: testLogLevel,
 		})
 		// Create a bunch of tasks
 		for i := 0; i < count; i++ {
@@ -60,7 +60,6 @@ func BenchmarkEndToEnd(b *testing.B) {
 	const count = 100000
 	for n := 0; n < b.N; n++ {
 		b.StopTimer() // begin setup
-		rand.Seed(time.Now().UnixNano())
 		setup(b)
 		redis := &RedisClientOpt{
 			Addr: redisAddr,
@@ -72,6 +71,7 @@ func BenchmarkEndToEnd(b *testing.B) {
 			RetryDelayFunc: func(n int, err error, t *Task) time.Duration {
 				return time.Second
 			},
+			LogLevel: testLogLevel,
 		})
 		// Create a bunch of tasks
 		for i := 0; i < count; i++ {
@@ -90,8 +90,16 @@ func BenchmarkEndToEnd(b *testing.B) {
 		var wg sync.WaitGroup
 		wg.Add(count * 2)
 		handler := func(ctx context.Context, t *Task) error {
-			// randomly fail 1% of tasks
-			if rand.Intn(100) == 1 {
+			n, err := t.Payload.GetInt("data")
+			if err != nil {
+				b.Logf("internal error: %v", err)
+			}
+			retried, ok := GetRetryCount(ctx)
+			if !ok {
+				b.Logf("internal error: %v", err)
+			}
+			// Fail 1% of tasks for the first attempt.
+			if retried == 0 && n%100 == 0 {
 				return fmt.Errorf(":(")
 			}
 			wg.Done()
@@ -131,6 +139,7 @@ func BenchmarkEndToEndMultipleQueues(b *testing.B) {
 				"default": 3,
 				"low":     1,
 			},
+			LogLevel: testLogLevel,
 		})
 		// Create a bunch of tasks
 		for i := 0; i < highCount; i++ {
@@ -168,3 +177,62 @@ func BenchmarkEndToEndMultipleQueues(b *testing.B) {
 		b.StartTimer() // end teardown
 	}
 }
+
+// E2E benchmark to check client enqueue operation performs correctly,
+// while server is busy processing tasks.
+func BenchmarkClientWhileServerRunning(b *testing.B) {
+	const count = 10000
+	for n := 0; n < b.N; n++ {
+		b.StopTimer() // begin setup
+		setup(b)
+		redis := &RedisClientOpt{
+			Addr: redisAddr,
+			DB:   redisDB,
+		}
+		client := NewClient(redis)
+		srv := NewServer(redis, Config{
+			Concurrency: 10,
+			RetryDelayFunc: func(n int, err error, t *Task) time.Duration {
+				return time.Second
+			},
+			LogLevel: testLogLevel,
+		})
+		// Enqueue 10,000 tasks.
+		for i := 0; i < count; i++ {
+			t := NewTask(fmt.Sprintf("task%d", i), map[string]interface{}{"data": i})
+			if err := client.Enqueue(t); err != nil {
+				b.Fatalf("could not enqueue a task: %v", err)
+			}
+		}
+		// Schedule 10,000 tasks.
+		for i := 0; i < count; i++ {
+			t := NewTask(fmt.Sprintf("scheduled%d", i), map[string]interface{}{"data": i})
+			if err := client.EnqueueAt(time.Now().Add(time.Second), t); err != nil {
+				b.Fatalf("could not enqueue a task: %v", err)
+			}
+		}
+
+		handler := func(ctx context.Context, t *Task) error {
+			return nil
+		}
+		srv.Start(HandlerFunc(handler))
+
+		b.StartTimer() // end setup
+
+		b.Log("Starting enqueueing")
+		enqueued := 0
+		for enqueued < 100000 {
+			t := NewTask(fmt.Sprintf("enqueued%d", enqueued), map[string]interface{}{"data": enqueued})
+			if err := client.Enqueue(t); err != nil {
+				b.Logf("could not enqueue task %d: %v", enqueued, err)
+				continue
+			}
+			enqueued++
+		}
+		b.Logf("Finished enqueueing %d tasks", enqueued)
+
+		b.StopTimer() // begin teardown
+		srv.Stop()
+		b.StartTimer() // end teardown
+	}
+}

client.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/hibiken/asynq/internal/base"
@@ -23,13 +24,18 @@ import (
 //
 // Clients are safe for concurrent use by multiple goroutines.
 type Client struct {
+	mu   sync.Mutex
+	opts map[string][]Option
 	rdb  *rdb.RDB
 }
 
 // NewClient and returns a new Client given a redis connection option.
 func NewClient(r RedisConnOpt) *Client {
 	rdb := rdb.NewRDB(createRedisClient(r))
-	return &Client{rdb}
+	return &Client{
+		opts: make(map[string][]Option),
+		rdb:  rdb,
+	}
 }
 
 // Option specifies the task processing behavior.
@@ -159,10 +165,19 @@ func serializePayload(payload map[string]interface{}) string {
 	return b.String()
 }
 
-const (
-	// Max retry count by default
-	defaultMaxRetry = 25
-)
+// Default max retry count used if nothing is specified.
+const defaultMaxRetry = 25
+
+// SetDefaultOptions sets options to be used for a given task type.
+// The argument opts specifies the behavior of task processing.
+// If there are conflicting Option values the last one overrides others.
+//
+// Default options can be overridden by options passed at enqueue time.
+func (c *Client) SetDefaultOptions(taskType string, opts ...Option) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.opts[taskType] = opts
+}
 
 // EnqueueAt schedules task to be enqueued at the specified time.
 //
@@ -171,6 +186,40 @@ const (
 // The argument opts specifies the behavior of task processing.
 // If there are conflicting Option values the last one overrides others.
 func (c *Client) EnqueueAt(t time.Time, task *Task, opts ...Option) error {
+	return c.enqueueAt(t, task, opts...)
+}
+
+// Enqueue enqueues task to be processed immediately.
+//
+// Enqueue returns nil if the task is enqueued successfully, otherwise returns a non-nil error.
+//
+// The argument opts specifies the behavior of task processing.
+// If there are conflicting Option values the last one overrides others.
+func (c *Client) Enqueue(task *Task, opts ...Option) error {
+	return c.enqueueAt(time.Now(), task, opts...)
+}
+
+// EnqueueIn schedules task to be enqueued after the specified delay.
+//
+// EnqueueIn returns nil if the task is scheduled successfully, otherwise returns a non-nil error.
+//
+// The argument opts specifies the behavior of task processing.
+// If there are conflicting Option values the last one overrides others.
+func (c *Client) EnqueueIn(d time.Duration, task *Task, opts ...Option) error {
+	return c.enqueueAt(time.Now().Add(d), task, opts...)
+}
+
+// Close closes the connection with redis server.
+func (c *Client) Close() error {
+	return c.rdb.Close()
+}
+
+func (c *Client) enqueueAt(t time.Time, task *Task, opts ...Option) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if defaults, ok := c.opts[task.Type]; ok {
+		opts = append(defaults, opts...)
+	}
 	opt := composeOptions(opts...)
 	msg := &base.TaskMessage{
 		ID:        xid.New(),
@@ -194,26 +243,6 @@ func (c *Client) EnqueueAt(t time.Time, task *Task, opts ...Option) error {
 	return err
 }
 
-// Enqueue enqueues task to be processed immediately.
-//
-// Enqueue returns nil if the task is enqueued successfully, otherwise returns a non-nil error.
-//
-// The argument opts specifies the behavior of task processing.
-// If there are conflicting Option values the last one overrides others.
-func (c *Client) Enqueue(task *Task, opts ...Option) error {
-	return c.EnqueueAt(time.Now(), task, opts...)
-}
-
-// EnqueueIn schedules task to be enqueued after the specified delay.
-//
-// EnqueueIn returns nil if the task is scheduled successfully, otherwise returns a non-nil error.
-//
-// The argument opts specifies the behavior of task processing.
-// If there are conflicting Option values the last one overrides others.
-func (c *Client) EnqueueIn(d time.Duration, task *Task, opts ...Option) error {
-	return c.EnqueueAt(time.Now().Add(d), task, opts...)
-}
-
 func (c *Client) enqueue(msg *base.TaskMessage, uniqueTTL time.Duration) error {
 	if uniqueTTL > 0 {
 		return c.rdb.EnqueueUnique(msg, uniqueTTL)

client_test.go

@@ -15,6 +15,11 @@ import (
 	"github.com/hibiken/asynq/internal/base"
 )
 
+var (
+	noTimeout  = time.Duration(0).String()
+	noDeadline = time.Time{}.Format(time.RFC3339)
+)
+
 func TestClientEnqueueAt(t *testing.T) {
 	r := setup(t)
 	client := NewClient(RedisClientOpt{
@@ -27,9 +32,6 @@ func TestClientEnqueueAt(t *testing.T) {
 	var (
 		now          = time.Now()
 		oneHourLater = now.Add(time.Hour)
-
-		noTimeout  = time.Duration(0).String()
-		noDeadline = time.Time{}.Format(time.RFC3339)
 	)
 
 	tests := []struct {
@@ -113,11 +115,6 @@ func TestClientEnqueue(t *testing.T) {
 
 	task := NewTask("send_email", map[string]interface{}{"to": "customer@gmail.com", "from": "merchant@example.com"})
 
-	var (
-		noTimeout  = time.Duration(0).String()
-		noDeadline = time.Time{}.Format(time.RFC3339)
-	)
-
 	tests := []struct {
 		desc         string
 		task         *Task
@@ -287,11 +284,6 @@ func TestClientEnqueueIn(t *testing.T) {
 
 	task := NewTask("send_email", map[string]interface{}{"to": "customer@gmail.com", "from": "merchant@example.com"})
 
-	var (
-		noTimeout  = time.Duration(0).String()
-		noDeadline = time.Time{}.Format(time.RFC3339)
-	)
-
 	tests := []struct {
 		desc          string
 		task          *Task
@@ -364,6 +356,86 @@ func TestClientEnqueueIn(t *testing.T) {
 	}
 }
 
+func TestClientDefaultOptions(t *testing.T) {
+	r := setup(t)
+
+	tests := []struct {
+		desc        string
+		defaultOpts []Option // options set at the client level.
+		opts        []Option // options used at enqueue time.
+		task        *Task
+		queue       string // queue that the message should go into.
+		want        *base.TaskMessage
+	}{
+		{
+			desc:        "With queue routing option",
+			defaultOpts: []Option{Queue("feed")},
+			opts:        []Option{},
+			task:        NewTask("feed:import", nil),
+			queue:       "feed",
+			want: &base.TaskMessage{
+				Type:     "feed:import",
+				Payload:  nil,
+				Retry:    defaultMaxRetry,
+				Queue:    "feed",
+				Timeout:  noTimeout,
+				Deadline: noDeadline,
+			},
+		},
+		{
+			desc:        "With multiple options",
+			defaultOpts: []Option{Queue("feed"), MaxRetry(5)},
+			opts:        []Option{},
+			task:        NewTask("feed:import", nil),
+			queue:       "feed",
+			want: &base.TaskMessage{
+				Type:     "feed:import",
+				Payload:  nil,
+				Retry:    5,
+				Queue:    "feed",
+				Timeout:  noTimeout,
+				Deadline: noDeadline,
+			},
+		},
+		{
+			desc:        "With overriding options at enqueue time",
+			defaultOpts: []Option{Queue("feed"), MaxRetry(5)},
+			opts:        []Option{Queue("critical")},
+			task:        NewTask("feed:import", nil),
+			queue:       "critical",
+			want: &base.TaskMessage{
+				Type:     "feed:import",
+				Payload:  nil,
+				Retry:    5,
+				Queue:    "critical",
+				Timeout:  noTimeout,
+				Deadline: noDeadline,
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		h.FlushDB(t, r)
+		c := NewClient(RedisClientOpt{Addr: redisAddr, DB: redisDB})
+		c.SetDefaultOptions(tc.task.Type, tc.defaultOpts...)
+		err := c.Enqueue(tc.task, tc.opts...)
+		if err != nil {
+			t.Fatal(err)
+		}
+		enqueued := h.GetEnqueuedMessages(t, r, tc.queue)
+		if len(enqueued) != 1 {
+			t.Errorf("%s;\nexpected queue %q to have one message; got %d messages in the queue.",
+				tc.desc, tc.queue, len(enqueued))
+			continue
+		}
+		got := enqueued[0]
+		if diff := cmp.Diff(tc.want, got, h.IgnoreIDOpt); diff != "" {
+			t.Errorf("%s;\nmismatch found in enqueued task message; (-want,+got)\n%s",
+				tc.desc, diff)
+		}
+	}
+}
+
 func TestUniqueKey(t *testing.T) {
 	tests := []struct {
 		desc  string

context.go

@@ -0,0 +1,85 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package asynq
+
+import (
+	"context"
+	"time"
+
+	"github.com/hibiken/asynq/internal/base"
+)
+
+// A taskMetadata holds task scoped data to put in context.
+type taskMetadata struct {
+	id         string
+	maxRetry   int
+	retryCount int
+}
+
+// ctxKey type is unexported to prevent collisions with context keys defined in
+// other packages.
+type ctxKey int
+
+// metadataCtxKey is the context key for the task metadata.
+// Its value of zero is arbitrary.
+const metadataCtxKey ctxKey = 0
+
+// createContext returns a context and cancel function for a given task message.
+func createContext(msg *base.TaskMessage) (ctx context.Context, cancel context.CancelFunc) {
+	metadata := taskMetadata{
+		id:         msg.ID.String(),
+		maxRetry:   msg.Retry,
+		retryCount: msg.Retried,
+	}
+	ctx = context.WithValue(context.Background(), metadataCtxKey, metadata)
+	timeout, err := time.ParseDuration(msg.Timeout)
+	if err == nil && timeout != 0 {
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+	}
+	deadline, err := time.Parse(time.RFC3339, msg.Deadline)
+	if err == nil && !deadline.IsZero() {
+		ctx, cancel = context.WithDeadline(ctx, deadline)
+	}
+	if cancel == nil {
+		ctx, cancel = context.WithCancel(ctx)
+	}
+	return ctx, cancel
+}
+
+// GetTaskID extracts a task ID from a context, if any.
+//
+// ID of a task is guaranteed to be unique.
+// ID of a task doesn't change if the task is being retried.
+func GetTaskID(ctx context.Context) (id string, ok bool) {
+	metadata, ok := ctx.Value(metadataCtxKey).(taskMetadata)
+	if !ok {
+		return "", false
+	}
+	return metadata.id, true
+}
+
+// GetRetryCount extracts retry count from a context, if any.
+//
+// Return value n indicates the number of times associated task has been
+// retried so far.
+func GetRetryCount(ctx context.Context) (n int, ok bool) {
+	metadata, ok := ctx.Value(metadataCtxKey).(taskMetadata)
+	if !ok {
+		return 0, false
+	}
+	return metadata.retryCount, true
+}
+
+// GetMaxRetry extracts maximum retry from a context, if any.
+//
+// Return value n indicates the maximum number of times the assoicated task
+// can be retried if ProcessTask returns a non-nil error.
+func GetMaxRetry(ctx context.Context) (n int, ok bool) {
+	metadata, ok := ctx.Value(metadataCtxKey).(taskMetadata)
+	if !ok {
+		return 0, false
+	}
+	return metadata.maxRetry, true
+}

context_test.go

@@ -0,0 +1,157 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package asynq
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/hibiken/asynq/internal/base"
+	"github.com/rs/xid"
+)
+
+func TestCreateContextWithTimeRestrictions(t *testing.T) {
+	var (
+		noTimeout  = time.Duration(0)
+		noDeadline = time.Time{}
+	)
+
+	tests := []struct {
+		desc         string
+		timeout      time.Duration
+		deadline     time.Time
+		wantDeadline time.Time
+	}{
+		{"only with timeout", 10 * time.Second, noDeadline, time.Now().Add(10 * time.Second)},
+		{"only with deadline", noTimeout, time.Now().Add(time.Hour), time.Now().Add(time.Hour)},
+		{"with timeout and deadline (timeout < deadline)", 10 * time.Second, time.Now().Add(time.Hour), time.Now().Add(10 * time.Second)},
+		{"with timeout and deadline (timeout > deadline)", 10 * time.Minute, time.Now().Add(30 * time.Second), time.Now().Add(30 * time.Second)},
+	}
+
+	for _, tc := range tests {
+		msg := &base.TaskMessage{
+			Type:     "something",
+			ID:       xid.New(),
+			Timeout:  tc.timeout.String(),
+			Deadline: tc.deadline.Format(time.RFC3339),
+		}
+
+		ctx, cancel := createContext(msg)
+
+		select {
+		case x := <-ctx.Done():
+			t.Errorf("%s: <-ctx.Done() == %v, want nothing (it should block)", tc.desc, x)
+		default:
+		}
+
+		got, ok := ctx.Deadline()
+		if !ok {
+			t.Errorf("%s: ctx.Deadline() returned false, want deadline to be set", tc.desc)
+		}
+		if !cmp.Equal(tc.wantDeadline, got, cmpopts.EquateApproxTime(time.Second)) {
+			t.Errorf("%s: ctx.Deadline() returned %v, want %v", tc.desc, got, tc.wantDeadline)
+		}
+
+		cancel()
+
+		select {
+		case <-ctx.Done():
+		default:
+			t.Errorf("ctx.Done() blocked, want it to be non-blocking")
+		}
+	}
+}
+
+func TestCreateContextWithoutTimeRestrictions(t *testing.T) {
+	msg := &base.TaskMessage{
+		Type:     "something",
+		ID:       xid.New(),
+		Timeout:  time.Duration(0).String(),        // zero value to indicate no timeout
+		Deadline: time.Time{}.Format(time.RFC3339), // zero value to indicate no deadline
+	}
+
+	ctx, cancel := createContext(msg)
+
+	select {
+	case x := <-ctx.Done():
+		t.Errorf("<-ctx.Done() == %v, want nothing (it should block)", x)
+	default:
+	}
+
+	_, ok := ctx.Deadline()
+	if ok {
+		t.Error("ctx.Deadline() returned true, want deadline to not be set")
+	}
+
+	cancel()
+
+	select {
+	case <-ctx.Done():
+	default:
+		t.Error("ctx.Done() blocked, want it to be non-blocking")
+	}
+}
+
+func TestGetTaskMetadataFromContext(t *testing.T) {
+	tests := []struct {
+		desc string
+		msg  *base.TaskMessage
+	}{
+		{"with zero retried message", &base.TaskMessage{Type: "something", ID: xid.New(), Retry: 25, Retried: 0}},
+		{"with non-zero retried message", &base.TaskMessage{Type: "something", ID: xid.New(), Retry: 10, Retried: 5}},
+	}
+
+	for _, tc := range tests {
+		ctx, _ := createContext(tc.msg)
+
+		id, ok := GetTaskID(ctx)
+		if !ok {
+			t.Errorf("%s: GetTaskID(ctx) returned ok == false", tc.desc)
+		}
+		if ok && id != tc.msg.ID.String() {
+			t.Errorf("%s: GetTaskID(ctx) returned id == %q, want %q", tc.desc, id, tc.msg.ID.String())
+		}
+
+		retried, ok := GetRetryCount(ctx)
+		if !ok {
+			t.Errorf("%s: GetRetryCount(ctx) returned ok == false", tc.desc)
+		}
+		if ok && retried != tc.msg.Retried {
+			t.Errorf("%s: GetRetryCount(ctx) returned n == %d want %d", tc.desc, retried, tc.msg.Retried)
+		}
+
+		maxRetry, ok := GetMaxRetry(ctx)
+		if !ok {
+			t.Errorf("%s: GetMaxRetry(ctx) returned ok == false", tc.desc)
+		}
+		if ok && maxRetry != tc.msg.Retry {
+			t.Errorf("%s: GetMaxRetry(ctx) returned n == %d want %d", tc.desc, maxRetry, tc.msg.Retry)
+		}
+	}
+}
+
+func TestGetTaskMetadataFromContextError(t *testing.T) {
+	tests := []struct {
+		desc string
+		ctx  context.Context
+	}{
+		{"with background context", context.Background()},
+	}
+
+	for _, tc := range tests {
+		if _, ok := GetTaskID(tc.ctx); ok {
+			t.Errorf("%s: GetTaskID(ctx) returned ok == true", tc.desc)
+		}
+		if _, ok := GetRetryCount(tc.ctx); ok {
+			t.Errorf("%s: GetRetryCount(ctx) returned ok == true", tc.desc)
+		}
+		if _, ok := GetMaxRetry(tc.ctx); ok {
+			t.Errorf("%s: GetMaxRetry(ctx) returned ok == true", tc.desc)
+		}
+	}
+}

example_test.go

@@ -0,0 +1,95 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package asynq_test
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+
+	"github.com/hibiken/asynq"
+	"golang.org/x/sys/unix"
+)
+
+func ExampleServer_Run() {
+	srv := asynq.NewServer(
+		asynq.RedisClientOpt{Addr: ":6379"},
+		asynq.Config{Concurrency: 20},
+	)
+
+	h := asynq.NewServeMux()
+	// ... Register handlers
+
+	// Run blocks and waits for os signal to terminate the program.
+	if err := srv.Run(h); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func ExampleServer_Stop() {
+	srv := asynq.NewServer(
+		asynq.RedisClientOpt{Addr: ":6379"},
+		asynq.Config{Concurrency: 20},
+	)
+
+	h := asynq.NewServeMux()
+	// ... Register handlers
+
+	if err := srv.Start(h); err != nil {
+		log.Fatal(err)
+	}
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, unix.SIGTERM, unix.SIGINT)
+	<-sigs // wait for termination signal
+
+	srv.Stop()
+}
+
+func ExampleServer_Quiet() {
+	srv := asynq.NewServer(
+		asynq.RedisClientOpt{Addr: ":6379"},
+		asynq.Config{Concurrency: 20},
+	)
+
+	h := asynq.NewServeMux()
+	// ... Register handlers
+
+	if err := srv.Start(h); err != nil {
+		log.Fatal(err)
+	}
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, unix.SIGTERM, unix.SIGINT, unix.SIGTSTP)
+	// Handle SIGTERM, SIGINT to exit the program.
+	// Handle SIGTSTP to stop processing new tasks.
+	for {
+		s := <-sigs
+		if s == unix.SIGTSTP {
+			srv.Quiet() // stop processing new tasks
+			continue
+		}
+		break
+	}
+
+	srv.Stop()
+}
+
+func ExampleParseRedisURI() {
+	rconn, err := asynq.ParseRedisURI("redis://localhost:6379/10")
+	if err != nil {
+		log.Fatal(err)
+	}
+	r, ok := rconn.(asynq.RedisClientOpt)
+	if !ok {
+		log.Fatal("unexpected type")
+	}
+	fmt.Println(r.Addr)
+	fmt.Println(r.DB)
+	// Output:
+	// localhost:6379
+	// 10
+}

heartbeat.go

@@ -5,68 +5,161 @@
 package asynq
 
 import (
+	"os"
 	"sync"
 	"time"
 
 	"github.com/hibiken/asynq/internal/base"
+	"github.com/hibiken/asynq/internal/log"
+	"github.com/rs/xid"
 )
 
 // heartbeater is responsible for writing process info to redis periodically to
 // indicate that the background worker process is up.
 type heartbeater struct {
-	logger Logger
+	logger *log.Logger
 	broker base.Broker
 
-	ss *base.ServerState
-
 	// channel to communicate back to the long running "heartbeater" goroutine.
 	done chan struct{}
 
 	// interval between heartbeats.
 	interval time.Duration
+
+	// following fields are initialized at construction time and are immutable.
+	host           string
+	pid            int
+	serverID       string
+	concurrency    int
+	queues         map[string]int
+	strictPriority bool
+
+	// following fields are mutable and should be accessed only by the
+	// heartbeater goroutine. In other words, confine these variables
+	// to this goroutine only.
+	started time.Time
+	workers map[string]workerStat
+
+	// status is shared with other goroutine but is concurrency safe.
+	status *base.ServerStatus
+
+	// channels to receive updates on active workers.
+	starting <-chan *base.TaskMessage
+	finished <-chan *base.TaskMessage
+}
+
+type heartbeaterParams struct {
+	logger         *log.Logger
+	broker         base.Broker
+	interval       time.Duration
+	concurrency    int
+	queues         map[string]int
+	strictPriority bool
+	status         *base.ServerStatus
+	starting       <-chan *base.TaskMessage
+	finished       <-chan *base.TaskMessage
+}
+
+func newHeartbeater(params heartbeaterParams) *heartbeater {
+	host, err := os.Hostname()
+	if err != nil {
+		host = "unknown-host"
 	}
 
-func newHeartbeater(l Logger, b base.Broker, ss *base.ServerState, interval time.Duration) *heartbeater {
 	return &heartbeater{
-		logger:   l,
-		broker:   b,
-		ss:       ss,
+		logger:   params.logger,
+		broker:   params.broker,
 		done:     make(chan struct{}),
-		interval: interval,
+		interval: params.interval,
+
+		host:           host,
+		pid:            os.Getpid(),
+		serverID:       xid.New().String(),
+		concurrency:    params.concurrency,
+		queues:         params.queues,
+		strictPriority: params.strictPriority,
+
+		status:   params.status,
+		workers:  make(map[string]workerStat),
+		starting: params.starting,
+		finished: params.finished,
 	}
 }
 
 func (h *heartbeater) terminate() {
-	h.logger.Info("Heartbeater shutting down...")
+	h.logger.Debug("Heartbeater shutting down...")
 	// Signal the heartbeater goroutine to stop.
 	h.done <- struct{}{}
 }
 
+// A workerStat records the message a worker is working on
+// and the time the worker has started processing the message.
+type workerStat struct {
+	started time.Time
+	msg     *base.TaskMessage
+}
+
 func (h *heartbeater) start(wg *sync.WaitGroup) {
-	h.ss.SetStarted(time.Now())
-	h.ss.SetStatus(base.StatusRunning)
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
+
+		h.started = time.Now()
+
 		h.beat()
+
+		timer := time.NewTimer(h.interval)
 		for {
 			select {
 			case <-h.done:
-				h.broker.ClearServerState(h.ss)
-				h.logger.Info("Heartbeater done")
+				h.broker.ClearServerState(h.host, h.pid, h.serverID)
+				h.logger.Debug("Heartbeater done")
+				timer.Stop()
 				return
-			case <-time.After(h.interval):
+
+			case <-timer.C:
 				h.beat()
+				timer.Reset(h.interval)
+
+			case msg := <-h.starting:
+				h.workers[msg.ID.String()] = workerStat{time.Now(), msg}
+
+			case msg := <-h.finished:
+				delete(h.workers, msg.ID.String())
 			}
 		}
 	}()
 }
 
 func (h *heartbeater) beat() {
+	info := base.ServerInfo{
+		Host:              h.host,
+		PID:               h.pid,
+		ServerID:          h.serverID,
+		Concurrency:       h.concurrency,
+		Queues:            h.queues,
+		StrictPriority:    h.strictPriority,
+		Status:            h.status.String(),
+		Started:           h.started,
+		ActiveWorkerCount: len(h.workers),
+	}
+
+	var ws []*base.WorkerInfo
+	for id, stat := range h.workers {
+		ws = append(ws, &base.WorkerInfo{
+			Host:    h.host,
+			PID:     h.pid,
+			ID:      id,
+			Type:    stat.msg.Type,
+			Queue:   stat.msg.Queue,
+			Payload: stat.msg.Payload,
+			Started: stat.started,
+		})
+	}
+
 	// Note: Set TTL to be long enough so that it won't expire before we write again
 	// and short enough to expire quickly once the process is shut down or killed.
-	err := h.broker.WriteServerState(h.ss, h.interval*2)
-	if err != nil {
-		h.logger.Error("could not write heartbeat data: %v", err)
+	if err := h.broker.WriteServerState(&info, ws, h.interval*2); err != nil {
+		h.logger.Errorf("could not write server state data: %v", err)
 	}
 }

heartbeat_test.go

@@ -37,9 +37,24 @@ func TestHeartbeater(t *testing.T) {
 	for _, tc := range tests {
 		h.FlushDB(t, r)
 
-		state := base.NewServerState(tc.host, tc.pid, tc.concurrency, tc.queues, false)
-		hb := newHeartbeater(testLogger, rdbClient, state, tc.interval)
+		status := base.NewServerStatus(base.StatusIdle)
+		hb := newHeartbeater(heartbeaterParams{
+			logger:         testLogger,
+			broker:         rdbClient,
+			interval:       tc.interval,
+			concurrency:    tc.concurrency,
+			queues:         tc.queues,
+			strictPriority: false,
+			status:         status,
+			starting:       make(chan *base.TaskMessage),
+			finished:       make(chan *base.TaskMessage),
+		})
 
+		// Change host and pid fields for testing purpose.
+		hb.host = tc.host
+		hb.pid = tc.pid
+
+		status.Set(base.StatusRunning)
 		var wg sync.WaitGroup
 		hb.start(&wg)
 
@@ -75,7 +90,7 @@ func TestHeartbeater(t *testing.T) {
 		}
 
 		// status change
-		state.SetStatus(base.StatusStopped)
+		status.Set(base.StatusStopped)
 
 		// allow for heartbeater to write to redis
 		time.Sleep(tc.interval * 2)
@@ -114,8 +129,17 @@ func TestHeartbeaterWithRedisDown(t *testing.T) {
 	}()
 	r := rdb.NewRDB(setup(t))
 	testBroker := testbroker.NewTestBroker(r)
-	ss := base.NewServerState("localhost", 1234, 10, map[string]int{"default": 1}, false)
-	hb := newHeartbeater(testLogger, testBroker, ss, time.Second)
+	hb := newHeartbeater(heartbeaterParams{
+		logger:         testLogger,
+		broker:         testBroker,
+		interval:       time.Second,
+		concurrency:    10,
+		queues:         map[string]int{"default": 1},
+		strictPriority: false,
+		status:         base.NewServerStatus(base.StatusRunning),
+		starting:       make(chan *base.TaskMessage),
+		finished:       make(chan *base.TaskMessage),
+	})
 
 	testBroker.Sleep()
 	var wg sync.WaitGroup

internal/asynqtest/asynqtest.go

@@ -57,7 +57,7 @@ var SortServerInfoOpt = cmp.Transformer("SortServerInfo", func(in []*base.Server
 var SortWorkerInfoOpt = cmp.Transformer("SortWorkerInfo", func(in []*base.WorkerInfo) []*base.WorkerInfo {
 	out := append([]*base.WorkerInfo(nil), in...) // Copy input to avoid mutating it
 	sort.Slice(out, func(i, j int) bool {
-		return out[i].ID.String() < out[j].ID.String()
+		return out[i].ID < out[j].ID
 	})
 	return out
 })

internal/base/base.go

@@ -34,6 +34,7 @@ const (
 	RetryQueue      = "asynq:retry"                  // ZSET
 	DeadQueue       = "asynq:dead"                   // ZSET
 	InProgressQueue = "asynq:in_progress"            // LIST
+	PausedQueues    = "asynq:paused"                 // SET
 	CancelChannel   = "asynq:cancel"                 // PubSub channel
 )
 
@@ -105,28 +106,23 @@ type TaskMessage struct {
 	UniqueKey string
 }
 
-// ServerState holds process level information.
-//
-// ServerStates are safe for concurrent use by multiple goroutines.
-type ServerState struct {
-	mu             sync.Mutex // guards all data fields
-	id             xid.ID
-	concurrency    int
-	queues         map[string]int
-	strictPriority bool
-	pid            int
-	host           string
-	status         ServerStatus
-	started        time.Time
-	workers        map[string]*workerStats
+// ServerStatus represents status of a server.
+// ServerStatus methods are concurrency safe.
+type ServerStatus struct {
+	mu  sync.Mutex
+	val ServerStatusValue
 }
 
-// ServerStatus represents status of a server.
-type ServerStatus int
+// NewServerStatus returns a new status instance given an initial value.
+func NewServerStatus(v ServerStatusValue) *ServerStatus {
+	return &ServerStatus{val: v}
+}
+
+type ServerStatusValue int
 
 const (
 	// StatusIdle indicates the server is in idle state.
-	StatusIdle ServerStatus = iota
+	StatusIdle ServerStatusValue = iota
 
 	// StatusRunning indicates the servier is up and processing tasks.
 	StatusRunning
@@ -145,117 +141,28 @@ var statuses = []string{
 	"stopped",
 }
 
-func (s ServerStatus) String() string {
-	if StatusIdle <= s && s <= StatusStopped {
-		return statuses[s]
+func (s *ServerStatus) String() string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if StatusIdle <= s.val && s.val <= StatusStopped {
+		return statuses[s.val]
 	}
 	return "unknown status"
 }
 
-type workerStats struct {
-	msg     *TaskMessage
-	started time.Time
+// Get returns the status value.
+func (s *ServerStatus) Get() ServerStatusValue {
+	s.mu.Lock()
+	v := s.val
+	s.mu.Unlock()
+	return v
 }
 
-// NewServerState returns a new instance of ServerState.
-func NewServerState(host string, pid, concurrency int, queues map[string]int, strict bool) *ServerState {
-	return &ServerState{
-		host:           host,
-		pid:            pid,
-		id:             xid.New(),
-		concurrency:    concurrency,
-		queues:         cloneQueueConfig(queues),
-		strictPriority: strict,
-		status:         StatusIdle,
-		workers:        make(map[string]*workerStats),
-	}
-}
-
-// SetStatus updates the status of server.
-func (ss *ServerState) SetStatus(status ServerStatus) {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	ss.status = status
-}
-
-// Status returns the status of server.
-func (ss *ServerState) Status() ServerStatus {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	return ss.status
-}
-
-// SetStarted records when the process started processing.
-func (ss *ServerState) SetStarted(t time.Time) {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	ss.started = t
-}
-
-// AddWorkerStats records when a worker started and which task it's processing.
-func (ss *ServerState) AddWorkerStats(msg *TaskMessage, started time.Time) {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	ss.workers[msg.ID.String()] = &workerStats{msg, started}
-}
-
-// DeleteWorkerStats removes a worker's entry from the process state.
-func (ss *ServerState) DeleteWorkerStats(msg *TaskMessage) {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	delete(ss.workers, msg.ID.String())
-}
-
-// GetInfo returns current state of server as a ServerInfo.
-func (ss *ServerState) GetInfo() *ServerInfo {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	return &ServerInfo{
-		Host:              ss.host,
-		PID:               ss.pid,
-		ServerID:          ss.id.String(),
-		Concurrency:       ss.concurrency,
-		Queues:            cloneQueueConfig(ss.queues),
-		StrictPriority:    ss.strictPriority,
-		Status:            ss.status.String(),
-		Started:           ss.started,
-		ActiveWorkerCount: len(ss.workers),
-	}
-}
-
-// GetWorkers returns a list of currently running workers' info.
-func (ss *ServerState) GetWorkers() []*WorkerInfo {
-	ss.mu.Lock()
-	defer ss.mu.Unlock()
-	var res []*WorkerInfo
-	for _, w := range ss.workers {
-		res = append(res, &WorkerInfo{
-			Host:    ss.host,
-			PID:     ss.pid,
-			ID:      w.msg.ID,
-			Type:    w.msg.Type,
-			Queue:   w.msg.Queue,
-			Payload: clonePayload(w.msg.Payload),
-			Started: w.started,
-		})
-	}
-	return res
-}
-
-func cloneQueueConfig(qcfg map[string]int) map[string]int {
-	res := make(map[string]int)
-	for qname, n := range qcfg {
-		res[qname] = n
-	}
-	return res
-}
-
-func clonePayload(payload map[string]interface{}) map[string]interface{} {
-	res := make(map[string]interface{})
-	for k, v := range payload {
-		res[k] = v
-	}
-	return res
+// Set sets the status value.
+func (s *ServerStatus) Set(v ServerStatusValue) {
+	s.mu.Lock()
+	s.val = v
+	s.mu.Unlock()
 }
 
 // ServerInfo holds information about a running server.
@@ -275,7 +182,7 @@ type ServerInfo struct {
 type WorkerInfo struct {
 	Host    string
 	PID     int
-	ID      xid.ID
+	ID      string
 	Type    string
 	Queue   string
 	Payload map[string]interface{}
@@ -344,9 +251,9 @@ type Broker interface {
 	Retry(msg *TaskMessage, processAt time.Time, errMsg string) error
 	Kill(msg *TaskMessage, errMsg string) error
 	RequeueAll() (int64, error)
-	CheckAndEnqueue(qnames ...string) error
-	WriteServerState(ss *ServerState, ttl time.Duration) error
-	ClearServerState(ss *ServerState) error
+	CheckAndEnqueue() error
+	WriteServerState(info *ServerInfo, workers []*WorkerInfo, ttl time.Duration) error
+	ClearServerState(host string, pid int, serverID string) error
 	CancelationPubSub() (*redis.PubSub, error) // TODO: Need to decouple from redis to support other brokers
 	PublishCancelation(id string) error
 	Close() error

internal/base/base_test.go

@@ -6,14 +6,9 @@ package base
 
 import (
 	"context"
-	"math/rand"
 	"sync"
 	"testing"
 	"time"
-
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
-	"github.com/rs/xid"
 )
 
 func TestQueueKey(t *testing.T) {
@@ -82,7 +77,8 @@ func TestServerInfoKey(t *testing.T) {
 	for _, tc := range tests {
 		got := ServerInfoKey(tc.hostname, tc.pid, tc.sid)
 		if got != tc.want {
-			t.Errorf("ServerInfoKey(%q, %d) = %q, want %q", tc.hostname, tc.pid, got, tc.want)
+			t.Errorf("ServerInfoKey(%q, %d, %q) = %q, want %q",
+				tc.hostname, tc.pid, tc.sid, got, tc.want)
 		}
 	}
 }
@@ -101,74 +97,34 @@ func TestWorkersKey(t *testing.T) {
 	for _, tc := range tests {
 		got := WorkersKey(tc.hostname, tc.pid, tc.sid)
 		if got != tc.want {
-			t.Errorf("WorkersKey(%q, %d) = %q, want = %q", tc.hostname, tc.pid, got, tc.want)
+			t.Errorf("WorkersKey(%q, %d, %q) = %q, want = %q",
+				tc.hostname, tc.pid, tc.sid, got, tc.want)
 		}
 	}
 }
 
-// Test for server state being accessed by multiple goroutines.
+// Test for status being accessed by multiple goroutines.
 // Run with -race flag to check for data race.
-func TestServerStateConcurrentAccess(t *testing.T) {
-	ss := NewServerState("127.0.0.1", 1234, 10, map[string]int{"default": 1}, false)
-	var wg sync.WaitGroup
-	started := time.Now()
-	msgs := []*TaskMessage{
-		{ID: xid.New(), Type: "type1", Payload: map[string]interface{}{"user_id": 42}},
-		{ID: xid.New(), Type: "type2"},
-		{ID: xid.New(), Type: "type3"},
-	}
+func TestStatusConcurrentAccess(t *testing.T) {
+	status := NewServerStatus(StatusIdle)
+
+	var wg sync.WaitGroup
 
-	// Simulate hearbeater calling SetStatus and SetStarted.
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		ss.SetStarted(started)
-		ss.SetStatus(StatusRunning)
-		if status := ss.Status(); status != StatusRunning {
-			t.Errorf("(*ServerState).Status() = %v, want %v", status, StatusRunning)
-		}
+		status.Get()
+		status.String()
 	}()
 
-	// Simulate processor starting worker goroutines.
-	for _, msg := range msgs {
-		wg.Add(1)
-		ss.AddWorkerStats(msg, time.Now())
-		go func(msg *TaskMessage) {
-			defer wg.Done()
-			time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond)
-			ss.DeleteWorkerStats(msg)
-		}(msg)
-	}
-
-	// Simulate hearbeater calling Get and GetWorkers
 	wg.Add(1)
 	go func() {
-		wg.Done()
-		for i := 0; i < 5; i++ {
-			ss.GetInfo()
-			ss.GetWorkers()
-			time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
-		}
+		defer wg.Done()
+		status.Set(StatusStopped)
+		status.String()
 	}()
 
 	wg.Wait()
-
-	want := &ServerInfo{
-		Host:              "127.0.0.1",
-		PID:               1234,
-		Concurrency:       10,
-		Queues:            map[string]int{"default": 1},
-		StrictPriority:    false,
-		Status:            "running",
-		Started:           started,
-		ActiveWorkerCount: 0,
-	}
-
-	got := ss.GetInfo()
-	if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(ServerInfo{}, "ServerID")); diff != "" {
-		t.Errorf("(*ServerState).GetInfo() = %+v, want %+v; (-want,+got)\n%s",
-			got, want, diff)
-	}
 }
 
 // Test for cancelations being accessed by multiple goroutines.

internal/log/log.go

@@ -6,52 +6,210 @@
 package log
 
 import (
+	"fmt"
 	"io"
 	stdlog "log"
 	"os"
+	"sync"
 )
 
-// NewLogger creates and returns a new instance of Logger.
-func NewLogger(out io.Writer) *Logger {
-	return &Logger{
-		stdlog.New(out, "", stdlog.Ldate|stdlog.Ltime|stdlog.Lmicroseconds|stdlog.LUTC),
-	}
+// Base supports logging at various log levels.
+type Base interface {
+	// Debug logs a message at Debug level.
+	Debug(args ...interface{})
+
+	// Info logs a message at Info level.
+	Info(args ...interface{})
+
+	// Warn logs a message at Warning level.
+	Warn(args ...interface{})
+
+	// Error logs a message at Error level.
+	Error(args ...interface{})
+
+	// Fatal logs a message at Fatal level
+	// and process will exit with status set to 1.
+	Fatal(args ...interface{})
 }
 
-// Logger is a wrapper object around log.Logger from the standard library.
+// baseLogger is a wrapper object around log.Logger from the standard library.
 // It supports logging at various log levels.
-type Logger struct {
+type baseLogger struct {
 	*stdlog.Logger
 }
 
 // Debug logs a message at Debug level.
-func (l *Logger) Debug(format string, args ...interface{}) {
-	format = "DEBUG: " + format
-	l.Printf(format, args...)
+func (l *baseLogger) Debug(args ...interface{}) {
+	l.prefixPrint("DEBUG: ", args...)
 }
 
 // Info logs a message at Info level.
-func (l *Logger) Info(format string, args ...interface{}) {
-	format = "INFO: " + format
-	l.Printf(format, args...)
+func (l *baseLogger) Info(args ...interface{}) {
+	l.prefixPrint("INFO: ", args...)
 }
 
 // Warn logs a message at Warning level.
-func (l *Logger) Warn(format string, args ...interface{}) {
-	format = "WARN: " + format
-	l.Printf(format, args...)
+func (l *baseLogger) Warn(args ...interface{}) {
+	l.prefixPrint("WARN: ", args...)
 }
 
 // Error logs a message at Error level.
-func (l *Logger) Error(format string, args ...interface{}) {
-	format = "ERROR: " + format
-	l.Printf(format, args...)
+func (l *baseLogger) Error(args ...interface{}) {
+	l.prefixPrint("ERROR: ", args...)
 }
 
 // Fatal logs a message at Fatal level
 // and process will exit with status set to 1.
-func (l *Logger) Fatal(format string, args ...interface{}) {
-	format = "FATAL: " + format
-	l.Printf(format, args...)
+func (l *baseLogger) Fatal(args ...interface{}) {
+	l.prefixPrint("FATAL: ", args...)
 	os.Exit(1)
 }
+
+func (l *baseLogger) prefixPrint(prefix string, args ...interface{}) {
+	args = append([]interface{}{prefix}, args...)
+	l.Print(args...)
+}
+
+// newBase creates and returns a new instance of baseLogger.
+func newBase(out io.Writer) *baseLogger {
+	prefix := fmt.Sprintf("asynq: pid=%d ", os.Getpid())
+	return &baseLogger{
+		stdlog.New(out, prefix, stdlog.Ldate|stdlog.Ltime|stdlog.Lmicroseconds|stdlog.LUTC),
+	}
+}
+
+// NewLogger creates and returns a new instance of Logger.
+// Log level is set to DebugLevel by default.
+func NewLogger(base Base) *Logger {
+	if base == nil {
+		base = newBase(os.Stderr)
+	}
+	return &Logger{base: base, level: DebugLevel}
+}
+
+// Logger logs message to io.Writer at various log levels.
+type Logger struct {
+	base Base
+
+	mu sync.Mutex
+	// Minimum log level for this logger.
+	// Message with level lower than this level won't be outputted.
+	level Level
+}
+
+// Level represents a log level.
+type Level int32
+
+const (
+	// DebugLevel is the lowest level of logging.
+	// Debug logs are intended for debugging and development purposes.
+	DebugLevel Level = iota
+
+	// InfoLevel is used for general informational log messages.
+	InfoLevel
+
+	// WarnLevel is used for undesired but relatively expected events,
+	// which may indicate a problem.
+	WarnLevel
+
+	// ErrorLevel is used for undesired and unexpected events that
+	// the program can recover from.
+	ErrorLevel
+
+	// FatalLevel is used for undesired and unexpected events that
+	// the program cannot recover from.
+	FatalLevel
+)
+
+// String is part of the fmt.Stringer interface.
+//
+// Used for testing and debugging purposes.
+func (l Level) String() string {
+	switch l {
+	case DebugLevel:
+		return "debug"
+	case InfoLevel:
+		return "info"
+	case WarnLevel:
+		return "warning"
+	case ErrorLevel:
+		return "error"
+	case FatalLevel:
+		return "fatal"
+	default:
+		return "unknown"
+	}
+}
+
+// canLogAt reports whether logger can log at level v.
+func (l *Logger) canLogAt(v Level) bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return v >= l.level
+}
+
+func (l *Logger) Debug(args ...interface{}) {
+	if !l.canLogAt(DebugLevel) {
+		return
+	}
+	l.base.Debug(args...)
+}
+
+func (l *Logger) Info(args ...interface{}) {
+	if !l.canLogAt(InfoLevel) {
+		return
+	}
+	l.base.Info(args...)
+}
+
+func (l *Logger) Warn(args ...interface{}) {
+	if !l.canLogAt(WarnLevel) {
+		return
+	}
+	l.base.Warn(args...)
+}
+
+func (l *Logger) Error(args ...interface{}) {
+	if !l.canLogAt(ErrorLevel) {
+		return
+	}
+	l.base.Error(args...)
+}
+
+func (l *Logger) Fatal(args ...interface{}) {
+	if !l.canLogAt(FatalLevel) {
+		return
+	}
+	l.base.Fatal(args...)
+}
+
+func (l *Logger) Debugf(format string, args ...interface{}) {
+	l.Debug(fmt.Sprintf(format, args...))
+}
+
+func (l *Logger) Infof(format string, args ...interface{}) {
+	l.Info(fmt.Sprintf(format, args...))
+}
+
+func (l *Logger) Warnf(format string, args ...interface{}) {
+	l.Warn(fmt.Sprintf(format, args...))
+}
+
+func (l *Logger) Errorf(format string, args ...interface{}) {
+	l.Error(fmt.Sprintf(format, args...))
+}
+
+func (l *Logger) Fatalf(format string, args ...interface{}) {
+	l.Fatal(fmt.Sprintf(format, args...))
+}
+
+// SetLevel sets the logger level.
+// It panics if v is less than DebugLevel or greater than FatalLevel.
+func (l *Logger) SetLevel(v Level) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if v < DebugLevel || v > FatalLevel {
+		panic("log: invalid log level")
+	}
+	l.level = v
+}

internal/log/log_test.go

@@ -13,6 +13,7 @@ import (
 
 // regexp for timestamps
 const (
+	rgxPID          = `[0-9]+`
 	rgxdate         = `[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]`
 	rgxtime         = `[0-9][0-9]:[0-9][0-9]:[0-9][0-9]`
 	rgxmicroseconds = `\.[0-9][0-9][0-9][0-9][0-9][0-9]`
@@ -29,18 +30,20 @@ func TestLoggerDebug(t *testing.T) {
 		{
 			desc:    "without trailing newline, logger adds newline",
 			message: "hello, world!",
-			wantPattern: fmt.Sprintf("^%s %s%s DEBUG: hello, world!\n$", rgxdate, rgxtime, rgxmicroseconds),
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s DEBUG: hello, world!\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
 		},
 		{
 			desc:    "with trailing newline, logger preserves newline",
 			message: "hello, world!\n",
-			wantPattern: fmt.Sprintf("^%s %s%s DEBUG: hello, world!\n$", rgxdate, rgxtime, rgxmicroseconds),
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s DEBUG: hello, world!\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
 		},
 	}
 
 	for _, tc := range tests {
 		var buf bytes.Buffer
-		logger := NewLogger(&buf)
+		logger := NewLogger(newBase(&buf))
 
 		logger.Debug(tc.message)
 
@@ -50,7 +53,7 @@ func TestLoggerDebug(t *testing.T) {
 			t.Fatal("pattern did not compile:", err)
 		}
 		if !matched {
-			t.Errorf("logger.info(%q) outputted %q, should match pattern %q",
+			t.Errorf("logger.Debug(%q) outputted %q, should match pattern %q",
 				tc.message, got, tc.wantPattern)
 		}
 	}
@@ -61,18 +64,20 @@ func TestLoggerInfo(t *testing.T) {
 		{
 			desc:    "without trailing newline, logger adds newline",
 			message: "hello, world!",
-			wantPattern: fmt.Sprintf("^%s %s%s INFO: hello, world!\n$", rgxdate, rgxtime, rgxmicroseconds),
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s INFO: hello, world!\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
 		},
 		{
 			desc:    "with trailing newline, logger preserves newline",
 			message: "hello, world!\n",
-			wantPattern: fmt.Sprintf("^%s %s%s INFO: hello, world!\n$", rgxdate, rgxtime, rgxmicroseconds),
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s INFO: hello, world!\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
 		},
 	}
 
 	for _, tc := range tests {
 		var buf bytes.Buffer
-		logger := NewLogger(&buf)
+		logger := NewLogger(newBase(&buf))
 
 		logger.Info(tc.message)
 
@@ -82,7 +87,7 @@ func TestLoggerInfo(t *testing.T) {
 			t.Fatal("pattern did not compile:", err)
 		}
 		if !matched {
-			t.Errorf("logger.info(%q) outputted %q, should match pattern %q",
+			t.Errorf("logger.Info(%q) outputted %q, should match pattern %q",
 				tc.message, got, tc.wantPattern)
 		}
 	}
@@ -93,18 +98,20 @@ func TestLoggerWarn(t *testing.T) {
 		{
 			desc:    "without trailing newline, logger adds newline",
 			message: "hello, world!",
-			wantPattern: fmt.Sprintf("^%s %s%s WARN: hello, world!\n$", rgxdate, rgxtime, rgxmicroseconds),
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s WARN: hello, world!\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
 		},
 		{
 			desc:    "with trailing newline, logger preserves newline",
 			message: "hello, world!\n",
-			wantPattern: fmt.Sprintf("^%s %s%s WARN: hello, world!\n$", rgxdate, rgxtime, rgxmicroseconds),
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s WARN: hello, world!\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
 		},
 	}
 
 	for _, tc := range tests {
 		var buf bytes.Buffer
-		logger := NewLogger(&buf)
+		logger := NewLogger(newBase(&buf))
 
 		logger.Warn(tc.message)
 
@@ -114,7 +121,7 @@ func TestLoggerWarn(t *testing.T) {
 			t.Fatal("pattern did not compile:", err)
 		}
 		if !matched {
-			t.Errorf("logger.info(%q) outputted %q, should match pattern %q",
+			t.Errorf("logger.Warn(%q) outputted %q, should match pattern %q",
 				tc.message, got, tc.wantPattern)
 		}
 	}
@@ -125,18 +132,20 @@ func TestLoggerError(t *testing.T) {
 		{
 			desc:    "without trailing newline, logger adds newline",
 			message: "hello, world!",
-			wantPattern: fmt.Sprintf("^%s %s%s ERROR: hello, world!\n$", rgxdate, rgxtime, rgxmicroseconds),
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s ERROR: hello, world!\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
 		},
 		{
 			desc:    "with trailing newline, logger preserves newline",
 			message: "hello, world!\n",
-			wantPattern: fmt.Sprintf("^%s %s%s ERROR: hello, world!\n$", rgxdate, rgxtime, rgxmicroseconds),
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s ERROR: hello, world!\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
 		},
 	}
 
 	for _, tc := range tests {
 		var buf bytes.Buffer
-		logger := NewLogger(&buf)
+		logger := NewLogger(newBase(&buf))
 
 		logger.Error(tc.message)
 
@@ -146,8 +155,234 @@ func TestLoggerError(t *testing.T) {
 			t.Fatal("pattern did not compile:", err)
 		}
 		if !matched {
-			t.Errorf("logger.info(%q) outputted %q, should match pattern %q",
+			t.Errorf("logger.Error(%q) outputted %q, should match pattern %q",
 				tc.message, got, tc.wantPattern)
 		}
 	}
 }
+
+type formatTester struct {
+	desc        string
+	format      string
+	args        []interface{}
+	wantPattern string // regexp that log output must match
+}
+
+func TestLoggerDebugf(t *testing.T) {
+	tests := []formatTester{
+		{
+			desc:   "Formats message with DEBUG prefix",
+			format: "hello, %s!",
+			args:   []interface{}{"Gopher"},
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s DEBUG: hello, Gopher!\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
+		},
+	}
+
+	for _, tc := range tests {
+		var buf bytes.Buffer
+		logger := NewLogger(newBase(&buf))
+
+		logger.Debugf(tc.format, tc.args...)
+
+		got := buf.String()
+		matched, err := regexp.MatchString(tc.wantPattern, got)
+		if err != nil {
+			t.Fatal("pattern did not compile:", err)
+		}
+		if !matched {
+			t.Errorf("logger.Debugf(%q, %v) outputted %q, should match pattern %q",
+				tc.format, tc.args, got, tc.wantPattern)
+		}
+	}
+}
+
+func TestLoggerInfof(t *testing.T) {
+	tests := []formatTester{
+		{
+			desc:   "Formats message with INFO prefix",
+			format: "%d,%d,%d",
+			args:   []interface{}{1, 2, 3},
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s INFO: 1,2,3\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
+		},
+	}
+
+	for _, tc := range tests {
+		var buf bytes.Buffer
+		logger := NewLogger(newBase(&buf))
+
+		logger.Infof(tc.format, tc.args...)
+
+		got := buf.String()
+		matched, err := regexp.MatchString(tc.wantPattern, got)
+		if err != nil {
+			t.Fatal("pattern did not compile:", err)
+		}
+		if !matched {
+			t.Errorf("logger.Infof(%q, %v) outputted %q, should match pattern %q",
+				tc.format, tc.args, got, tc.wantPattern)
+		}
+	}
+}
+
+func TestLoggerWarnf(t *testing.T) {
+	tests := []formatTester{
+		{
+			desc:   "Formats message with WARN prefix",
+			format: "hello, %s",
+			args:   []interface{}{"Gophers"},
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s WARN: hello, Gophers\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
+		},
+	}
+
+	for _, tc := range tests {
+		var buf bytes.Buffer
+		logger := NewLogger(newBase(&buf))
+
+		logger.Warnf(tc.format, tc.args...)
+
+		got := buf.String()
+		matched, err := regexp.MatchString(tc.wantPattern, got)
+		if err != nil {
+			t.Fatal("pattern did not compile:", err)
+		}
+		if !matched {
+			t.Errorf("logger.Warnf(%q, %v) outputted %q, should match pattern %q",
+				tc.format, tc.args, got, tc.wantPattern)
+		}
+	}
+}
+
+func TestLoggerErrorf(t *testing.T) {
+	tests := []formatTester{
+		{
+			desc:   "Formats message with ERROR prefix",
+			format: "hello, %s",
+			args:   []interface{}{"Gophers"},
+			wantPattern: fmt.Sprintf("^asynq: pid=%s %s %s%s ERROR: hello, Gophers\n$",
+				rgxPID, rgxdate, rgxtime, rgxmicroseconds),
+		},
+	}
+
+	for _, tc := range tests {
+		var buf bytes.Buffer
+		logger := NewLogger(newBase(&buf))
+
+		logger.Errorf(tc.format, tc.args...)
+
+		got := buf.String()
+		matched, err := regexp.MatchString(tc.wantPattern, got)
+		if err != nil {
+			t.Fatal("pattern did not compile:", err)
+		}
+		if !matched {
+			t.Errorf("logger.Errorf(%q, %v) outputted %q, should match pattern %q",
+				tc.format, tc.args, got, tc.wantPattern)
+		}
+	}
+}
+
+func TestLoggerWithLowerLevels(t *testing.T) {
+	// Logger should not log messages at a level
+	// lower than the specified level.
+	tests := []struct {
+		level Level
+		op    string
+	}{
+		// with level one above
+		{InfoLevel, "Debug"},
+		{InfoLevel, "Debugf"},
+		{WarnLevel, "Info"},
+		{WarnLevel, "Infof"},
+		{ErrorLevel, "Warn"},
+		{ErrorLevel, "Warnf"},
+		{FatalLevel, "Error"},
+		{FatalLevel, "Errorf"},
+		// with skip level
+		{WarnLevel, "Debug"},
+		{ErrorLevel, "Infof"},
+	}
+
+	for _, tc := range tests {
+		var buf bytes.Buffer
+		logger := NewLogger(newBase(&buf))
+		logger.SetLevel(tc.level)
+
+		switch tc.op {
+		case "Debug":
+			logger.Debug("hello")
+		case "Debugf":
+			logger.Debugf("hello, %s", "world")
+		case "Info":
+			logger.Info("hello")
+		case "Infof":
+			logger.Infof("hello, %s", "world")
+		case "Warn":
+			logger.Warn("hello")
+		case "Warnf":
+			logger.Warnf("hello, %s", "world")
+		case "Error":
+			logger.Error("hello")
+		case "Errorf":
+			logger.Errorf("hello, %s", "world")
+		default:
+			t.Fatalf("unexpected op: %q", tc.op)
+		}
+
+		if buf.String() != "" {
+			t.Errorf("logger.%s outputted log message when level is set to %v", tc.op, tc.level)
+		}
+	}
+}
+
+func TestLoggerWithSameOrHigherLevels(t *testing.T) {
+	// Logger should log messages at a level
+	// same as or higher than the specified level.
+	tests := []struct {
+		level Level
+		op    string
+	}{
+		// same level
+		{DebugLevel, "Debug"},
+		{InfoLevel, "Infof"},
+		{WarnLevel, "Warn"},
+		{ErrorLevel, "Errorf"},
+		// higher level
+		{DebugLevel, "Info"},
+		{InfoLevel, "Warnf"},
+		{WarnLevel, "Error"},
+	}
+
+	for _, tc := range tests {
+		var buf bytes.Buffer
+		logger := NewLogger(newBase(&buf))
+		logger.SetLevel(tc.level)
+
+		switch tc.op {
+		case "Debug":
+			logger.Debug("hello")
+		case "Debugf":
+			logger.Debugf("hello, %s", "world")
+		case "Info":
+			logger.Info("hello")
+		case "Infof":
+			logger.Infof("hello, %s", "world")
+		case "Warn":
+			logger.Warn("hello")
+		case "Warnf":
+			logger.Warnf("hello, %s", "world")
+		case "Error":
+			logger.Error("hello")
+		case "Errorf":
+			logger.Errorf("hello, %s", "world")
+		default:
+			t.Fatalf("unexpected op: %q", tc.op)
+		}
+
+		if buf.String() == "" {
+			t.Errorf("logger.%s did not output log message when level is set to %v", tc.op, tc.level)
+		}
+	}
+}

internal/rdb/inspect.go

@@ -7,6 +7,7 @@ package rdb
 import (
 	"encoding/json"
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 
@@ -25,10 +26,24 @@ type Stats struct {
 	Dead       int
 	Processed  int
 	Failed     int
-	Queues     map[string]int // map of queue name to number of tasks in the queue (e.g., "default": 100, "critical": 20)
+	Queues     []*Queue
 	Timestamp  time.Time
 }
 
+// Queue represents a task queue.
+type Queue struct {
+	// Name of the queue (e.g. "default", "critical").
+	// Note: It doesn't include the prefix "asynq:queues:".
+	Name string
+
+	// Paused indicates whether the queue is paused.
+	// If true, tasks in the queue should not be processed.
+	Paused bool
+
+	// Size is the number of tasks in the queue.
+	Size int
+}
+
 // DailyStats holds aggregate data for a given day.
 type DailyStats struct {
 	Processed int
@@ -143,8 +158,12 @@ func (r *RDB) CurrentStats() (*Stats, error) {
 	if err != nil {
 		return nil, err
 	}
+	paused, err := r.client.SMembersMap(base.PausedQueues).Result()
+	if err != nil {
+		return nil, err
+	}
 	stats := &Stats{
-		Queues:    make(map[string]int),
+		Queues:    make([]*Queue, 0),
 		Timestamp: now,
 	}
 	for i := 0; i < len(data); i += 2 {
@@ -154,7 +173,14 @@ func (r *RDB) CurrentStats() (*Stats, error) {
 		switch {
 		case strings.HasPrefix(key, base.QueuePrefix):
 			stats.Enqueued += val
-			stats.Queues[strings.TrimPrefix(key, base.QueuePrefix)] = val
+			q := Queue{
+				Name: strings.TrimPrefix(key, base.QueuePrefix),
+				Size: val,
+			}
+			if _, exist := paused[key]; exist {
+				q.Paused = true
+			}
+			stats.Queues = append(stats.Queues, &q)
 		case key == base.InProgressQueue:
 			stats.InProgress = val
 		case key == base.ScheduledQueue:
@@ -169,6 +195,9 @@ func (r *RDB) CurrentStats() (*Stats, error) {
 			stats.Failed = val
 		}
 	}
+	sort.Slice(stats.Queues, func(i, j int) bool {
+		return stats.Queues[i].Name < stats.Queues[j].Name
+	})
 	return stats, nil
 }
 
@@ -830,3 +859,33 @@ func (r *RDB) ListWorkers() ([]*base.WorkerInfo, error) {
 	}
 	return workers, nil
 }
+
+// KEYS[1] -> asynq:paused
+// ARGV[1] -> asynq:queues:<qname> - queue to pause
+var pauseCmd = redis.NewScript(`
+local ismem = redis.call("SISMEMBER", KEYS[1], ARGV[1])
+if ismem == 1 then
+	return redis.error_reply("queue is already paused")
+end
+return redis.call("SADD", KEYS[1], ARGV[1])`)
+
+// Pause pauses processing of tasks from the given queue.
+func (r *RDB) Pause(qname string) error {
+	qkey := base.QueueKey(qname)
+	return pauseCmd.Run(r.client, []string{base.PausedQueues}, qkey).Err()
+}
+
+// KEYS[1] -> asynq:paused
+// ARGV[1] -> asynq:queues:<qname> - queue to unpause
+var unpauseCmd = redis.NewScript(`
+local ismem = redis.call("SISMEMBER", KEYS[1], ARGV[1])
+if ismem == 0 then
+	return redis.error_reply("queue is not paused")
+end
+return redis.call("SREM", KEYS[1], ARGV[1])`)
+
+// Unpause resumes processing of tasks from the given queue.
+func (r *RDB) Unpause(qname string) error {
+	qkey := base.QueueKey(qname)
+	return unpauseCmd.Run(r.client, []string{base.PausedQueues}, qkey).Err()
+}

internal/rdb/inspect_test.go

@@ -38,6 +38,7 @@ func TestCurrentStats(t *testing.T) {
 		processed  int
 		failed     int
 		allQueues  []interface{}
+		paused     []string
 		want       *Stats
 	}{
 		{
@@ -55,6 +56,7 @@ func TestCurrentStats(t *testing.T) {
 			processed: 120,
 			failed:    2,
 			allQueues: []interface{}{base.DefaultQueue, base.QueueKey("critical"), base.QueueKey("low")},
+			paused:    []string{},
 			want: &Stats{
 				Enqueued:   3,
 				InProgress: 1,
@@ -64,7 +66,12 @@ func TestCurrentStats(t *testing.T) {
 				Processed:  120,
 				Failed:     2,
 				Timestamp:  now,
-				Queues:     map[string]int{base.DefaultQueueName: 1, "critical": 1, "low": 1},
+				// Queues should be sorted by name.
+				Queues: []*Queue{
+					{Name: "critical", Paused: false, Size: 1},
+					{Name: "default", Paused: false, Size: 1},
+					{Name: "low", Paused: false, Size: 1},
+				},
 			},
 		},
 		{
@@ -82,6 +89,7 @@ func TestCurrentStats(t *testing.T) {
 			processed: 90,
 			failed:    10,
 			allQueues: []interface{}{base.DefaultQueue},
+			paused:    []string{},
 			want: &Stats{
 				Enqueued:   0,
 				InProgress: 0,
@@ -91,13 +99,52 @@ func TestCurrentStats(t *testing.T) {
 				Processed:  90,
 				Failed:     10,
 				Timestamp:  now,
-				Queues:     map[string]int{base.DefaultQueueName: 0},
+				Queues: []*Queue{
+					{Name: base.DefaultQueueName, Paused: false, Size: 0},
+				},
+			},
+		},
+		{
+			enqueued: map[string][]*base.TaskMessage{
+				base.DefaultQueueName: {m1},
+				"critical":            {m5},
+				"low":                 {m6},
+			},
+			inProgress: []*base.TaskMessage{m2},
+			scheduled: []h.ZSetEntry{
+				{Msg: m3, Score: float64(now.Add(time.Hour).Unix())},
+				{Msg: m4, Score: float64(now.Unix())}},
+			retry:     []h.ZSetEntry{},
+			dead:      []h.ZSetEntry{},
+			processed: 120,
+			failed:    2,
+			allQueues: []interface{}{base.DefaultQueue, base.QueueKey("critical"), base.QueueKey("low")},
+			paused:    []string{"critical", "low"},
+			want: &Stats{
+				Enqueued:   3,
+				InProgress: 1,
+				Scheduled:  2,
+				Retry:      0,
+				Dead:       0,
+				Processed:  120,
+				Failed:     2,
+				Timestamp:  now,
+				Queues: []*Queue{
+					{Name: "critical", Paused: true, Size: 1},
+					{Name: "default", Paused: false, Size: 1},
+					{Name: "low", Paused: true, Size: 1},
+				},
 			},
 		},
 	}
 
 	for _, tc := range tests {
 		h.FlushDB(t, r.client) // clean up db before each test case
+		for _, qname := range tc.paused {
+			if err := r.Pause(qname); err != nil {
+				t.Fatal(err)
+			}
+		}
 		for qname, msgs := range tc.enqueued {
 			h.SeedEnqueuedQueue(t, r.client, msgs, qname)
 		}
@@ -136,7 +183,7 @@ func TestCurrentStatsWithoutData(t *testing.T) {
 		Processed:  0,
 		Failed:     0,
 		Timestamp:  time.Now(),
-		Queues:     map[string]int{},
+		Queues:     make([]*Queue, 0),
 	}
 
 	got, err := r.CurrentStats()
@@ -658,12 +705,14 @@ func TestListRetry(t *testing.T) {
 func TestListRetryPagination(t *testing.T) {
 	r := setup(t)
 	// create 100 tasks with an increasing number of wait time.
+	now := time.Now()
+	var seed []h.ZSetEntry
 	for i := 0; i < 100; i++ {
 		msg := h.NewTaskMessage(fmt.Sprintf("task %d", i), nil)
-		if err := r.Retry(msg, time.Now().Add(time.Duration(i)*time.Second), "error"); err != nil {
-			t.Fatal(err)
-		}
+		processAt := now.Add(time.Duration(i) * time.Second)
+		seed = append(seed, h.ZSetEntry{Msg: msg, Score: float64(processAt.Unix())})
 	}
+	h.SeedRetryQueue(t, r.client, seed)
 
 	tests := []struct {
 		desc      string
@@ -2055,60 +2104,48 @@ func TestListServers(t *testing.T) {
 	r := setup(t)
 
 	started1 := time.Now().Add(-time.Hour)
-	ss1 := base.NewServerState("do.droplet1", 1234, 10, map[string]int{"default": 1}, false)
-	ss1.SetStarted(started1)
-	ss1.SetStatus(base.StatusRunning)
 	info1 := &base.ServerInfo{
-		Concurrency:       10,
-		Queues:            map[string]int{"default": 1},
 		Host:              "do.droplet1",
 		PID:               1234,
+		ServerID:          "server123",
+		Concurrency:       10,
+		Queues:            map[string]int{"default": 1},
 		Status:            "running",
 		Started:           started1,
 		ActiveWorkerCount: 0,
 	}
 
 	started2 := time.Now().Add(-2 * time.Hour)
-	ss2 := base.NewServerState("do.droplet2", 9876, 20, map[string]int{"email": 1}, false)
-	ss2.SetStarted(started2)
-	ss2.SetStatus(base.StatusStopped)
-	ss2.AddWorkerStats(h.NewTaskMessage("send_email", nil), time.Now())
 	info2 := &base.ServerInfo{
-		Concurrency:       20,
-		Queues:            map[string]int{"email": 1},
 		Host:              "do.droplet2",
 		PID:               9876,
+		ServerID:          "server456",
+		Concurrency:       20,
+		Queues:            map[string]int{"email": 1},
 		Status:            "stopped",
 		Started:           started2,
 		ActiveWorkerCount: 1,
 	}
 
 	tests := []struct {
-		serverStates []*base.ServerState
-		want         []*base.ServerInfo
+		data []*base.ServerInfo
 	}{
 		{
-			serverStates: []*base.ServerState{},
-			want:         []*base.ServerInfo{},
+			data: []*base.ServerInfo{},
 		},
 		{
-			serverStates: []*base.ServerState{ss1},
-			want:         []*base.ServerInfo{info1},
+			data: []*base.ServerInfo{info1},
 		},
 		{
-			serverStates: []*base.ServerState{ss1, ss2},
-			want:         []*base.ServerInfo{info1, info2},
+			data: []*base.ServerInfo{info1, info2},
 		},
 	}
 
-	ignoreOpt := cmpopts.IgnoreUnexported(base.ServerInfo{})
-	ignoreFieldOpt := cmpopts.IgnoreFields(base.ServerInfo{}, "ServerID")
-
 	for _, tc := range tests {
 		h.FlushDB(t, r.client)
 
-		for _, ss := range tc.serverStates {
-			if err := r.WriteServerState(ss, 5*time.Second); err != nil {
+		for _, info := range tc.data {
+			if err := r.WriteServerState(info, []*base.WorkerInfo{}, 5*time.Second); err != nil {
 				t.Fatal(err)
 			}
 		}
@@ -2117,9 +2154,9 @@ func TestListServers(t *testing.T) {
 		if err != nil {
 			t.Errorf("r.ListServers returned an error: %v", err)
 		}
-		if diff := cmp.Diff(tc.want, got, h.SortServerInfoOpt, ignoreOpt, ignoreFieldOpt); diff != "" {
+		if diff := cmp.Diff(tc.data, got, h.SortServerInfoOpt); diff != "" {
 			t.Errorf("r.ListServers returned %v, want %v; (-want,+got)\n%s",
-				got, tc.serverStates, diff)
+				got, tc.data, diff)
 		}
 	}
 }
@@ -2127,37 +2164,23 @@ func TestListServers(t *testing.T) {
 func TestListWorkers(t *testing.T) {
 	r := setup(t)
 
-	const (
+	var (
 		host = "127.0.0.1"
 		pid  = 4567
+
+		m1 = h.NewTaskMessage("send_email", map[string]interface{}{"user_id": "abc123"})
+		m2 = h.NewTaskMessage("gen_thumbnail", map[string]interface{}{"path": "some/path/to/image/file"})
+		m3 = h.NewTaskMessage("reindex", map[string]interface{}{})
 	)
 
-	m1 := h.NewTaskMessage("send_email", map[string]interface{}{"user_id": "abc123"})
-	m2 := h.NewTaskMessage("gen_thumbnail", map[string]interface{}{"path": "some/path/to/image/file"})
-	m3 := h.NewTaskMessage("reindex", map[string]interface{}{})
-	t1 := time.Now().Add(-time.Second)
-	t2 := time.Now().Add(-10 * time.Second)
-	t3 := time.Now().Add(-time.Minute)
-
-	type workerStats struct {
-		msg     *base.TaskMessage
-		started time.Time
-	}
-
 	tests := []struct {
-		workers []*workerStats
-		want    []*base.WorkerInfo
+		data []*base.WorkerInfo
 	}{
 		{
-			workers: []*workerStats{
-				{m1, t1},
-				{m2, t2},
-				{m3, t3},
-			},
-			want: []*base.WorkerInfo{
-				{Host: host, PID: pid, ID: m1.ID, Type: m1.Type, Queue: m1.Queue, Payload: m1.Payload, Started: t1},
-				{Host: host, PID: pid, ID: m2.ID, Type: m2.Type, Queue: m2.Queue, Payload: m2.Payload, Started: t2},
-				{Host: host, PID: pid, ID: m3.ID, Type: m3.Type, Queue: m3.Queue, Payload: m3.Payload, Started: t3},
+			data: []*base.WorkerInfo{
+				{Host: host, PID: pid, ID: m1.ID.String(), Type: m1.Type, Queue: m1.Queue, Payload: m1.Payload, Started: time.Now().Add(-1 * time.Second)},
+				{Host: host, PID: pid, ID: m2.ID.String(), Type: m2.Type, Queue: m2.Queue, Payload: m2.Payload, Started: time.Now().Add(-5 * time.Second)},
+				{Host: host, PID: pid, ID: m3.ID.String(), Type: m3.Type, Queue: m3.Queue, Payload: m3.Payload, Started: time.Now().Add(-30 * time.Second)},
 			},
 		},
 	}
@@ -2165,13 +2188,7 @@ func TestListWorkers(t *testing.T) {
 	for _, tc := range tests {
 		h.FlushDB(t, r.client)
 
-		ss := base.NewServerState(host, pid, 10, map[string]int{"default": 1}, false)
-
-		for _, w := range tc.workers {
-			ss.AddWorkerStats(w.msg, w.started)
-		}
-
-		err := r.WriteServerState(ss, time.Minute)
+		err := r.WriteServerState(&base.ServerInfo{}, tc.data, time.Minute)
 		if err != nil {
 			t.Errorf("could not write server state to redis: %v", err)
 			continue
@@ -2183,8 +2200,165 @@ func TestListWorkers(t *testing.T) {
 			continue
 		}
 
-		if diff := cmp.Diff(tc.want, got, h.SortWorkerInfoOpt); diff != "" {
-			t.Errorf("(*RDB).ListWorkers() = %v, want = %v; (-want,+got)\n%s", got, tc.want, diff)
+		if diff := cmp.Diff(tc.data, got, h.SortWorkerInfoOpt); diff != "" {
+			t.Errorf("(*RDB).ListWorkers() = %v, want = %v; (-want,+got)\n%s", got, tc.data, diff)
+		}
+	}
+}
+
+func TestPause(t *testing.T) {
+	r := setup(t)
+
+	tests := []struct {
+		initial []string // initial keys in the paused set
+		qname   string   // name of the queue to pause
+		want    []string // expected keys in the paused set
+	}{
+		{[]string{}, "default", []string{"asynq:queues:default"}},
+		{[]string{"asynq:queues:default"}, "critical", []string{"asynq:queues:default", "asynq:queues:critical"}},
+	}
+
+	for _, tc := range tests {
+		h.FlushDB(t, r.client)
+
+		// Set up initial state.
+		for _, qkey := range tc.initial {
+			if err := r.client.SAdd(base.PausedQueues, qkey).Err(); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		err := r.Pause(tc.qname)
+		if err != nil {
+			t.Errorf("Pause(%q) returned error: %v", tc.qname, err)
+		}
+
+		got, err := r.client.SMembers(base.PausedQueues).Result()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if diff := cmp.Diff(tc.want, got, h.SortStringSliceOpt); diff != "" {
+			t.Errorf("%q has members %v, want %v; (-want,+got)\n%s",
+				base.PausedQueues, got, tc.want, diff)
+		}
+	}
+}
+
+func TestPauseError(t *testing.T) {
+	r := setup(t)
+
+	tests := []struct {
+		desc    string   // test case description
+		initial []string // initial keys in the paused set
+		qname   string   // name of the queue to pause
+		want    []string // expected keys in the paused set
+	}{
+		{"queue already paused", []string{"asynq:queues:default"}, "default", []string{"asynq:queues:default"}},
+	}
+
+	for _, tc := range tests {
+		h.FlushDB(t, r.client)
+
+		// Set up initial state.
+		for _, qkey := range tc.initial {
+			if err := r.client.SAdd(base.PausedQueues, qkey).Err(); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		err := r.Pause(tc.qname)
+		if err == nil {
+			t.Errorf("%s; Pause(%q) returned nil: want error", tc.desc, tc.qname)
+		}
+
+		got, err := r.client.SMembers(base.PausedQueues).Result()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if diff := cmp.Diff(tc.want, got, h.SortStringSliceOpt); diff != "" {
+			t.Errorf("%s; %q has members %v, want %v; (-want,+got)\n%s",
+				tc.desc, base.PausedQueues, got, tc.want, diff)
+		}
+	}
+}
+
+func TestUnpause(t *testing.T) {
+	r := setup(t)
+
+	tests := []struct {
+		initial []string // initial keys in the paused set
+		qname   string   // name of the queue to unpause
+		want    []string // expected keys in the paused set
+	}{
+		{[]string{"asynq:queues:default"}, "default", []string{}},
+		{[]string{"asynq:queues:default", "asynq:queues:low"}, "low", []string{"asynq:queues:default"}},
+	}
+
+	for _, tc := range tests {
+		h.FlushDB(t, r.client)
+
+		// Set up initial state.
+		for _, qkey := range tc.initial {
+			if err := r.client.SAdd(base.PausedQueues, qkey).Err(); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		err := r.Unpause(tc.qname)
+		if err != nil {
+			t.Errorf("Unpause(%q) returned error: %v", tc.qname, err)
+		}
+
+		got, err := r.client.SMembers(base.PausedQueues).Result()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if diff := cmp.Diff(tc.want, got, h.SortStringSliceOpt); diff != "" {
+			t.Errorf("%q has members %v, want %v; (-want,+got)\n%s",
+				base.PausedQueues, got, tc.want, diff)
+		}
+	}
+}
+
+func TestUnpauseError(t *testing.T) {
+	r := setup(t)
+
+	tests := []struct {
+		desc    string   // test case description
+		initial []string // initial keys in the paused set
+		qname   string   // name of the queue to unpause
+		want    []string // expected keys in the paused set
+	}{
+		{"set is empty", []string{}, "default", []string{}},
+		{"queue is not in the set", []string{"asynq:queues:default"}, "low", []string{"asynq:queues:default"}},
+	}
+
+	for _, tc := range tests {
+		h.FlushDB(t, r.client)
+
+		// Set up initial state.
+		for _, qkey := range tc.initial {
+			if err := r.client.SAdd(base.PausedQueues, qkey).Err(); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		err := r.Unpause(tc.qname)
+		if err == nil {
+			t.Errorf("%s; Unpause(%q) returned nil: want error", tc.desc, tc.qname)
+		}
+
+		got, err := r.client.SMembers(base.PausedQueues).Result()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if diff := cmp.Diff(tc.want, got, h.SortStringSliceOpt); diff != "" {
+			t.Errorf("%s; %q has members %v, want %v; (-want,+got)\n%s",
+				tc.desc, base.PausedQueues, got, tc.want, diff)
 		}
 	}
 }

internal/rdb/rdb.go

@@ -103,19 +103,14 @@ func (r *RDB) EnqueueUnique(msg *base.TaskMessage, ttl time.Duration) error {
 }
 
 // Dequeue queries given queues in order and pops a task message if there is one and returns it.
+// Dequeue skips a queue if the queue is paused.
 // If all queues are empty, ErrNoProcessableTask error is returned.
 func (r *RDB) Dequeue(qnames ...string) (*base.TaskMessage, error) {
-	var data string
-	var err error
-	if len(qnames) == 1 {
-		data, err = r.dequeueSingle(base.QueueKey(qnames[0]))
-	} else {
-		var keys []string
+	var qkeys []interface{}
 	for _, q := range qnames {
-			keys = append(keys, base.QueueKey(q))
-		}
-		data, err = r.dequeue(keys...)
+		qkeys = append(qkeys, base.QueueKey(q))
 	}
+	data, err := r.dequeue(qkeys...)
 	if err == redis.Nil {
 		return nil, ErrNoProcessableTask
 	}
@@ -130,29 +125,26 @@ func (r *RDB) Dequeue(qnames ...string) (*base.TaskMessage, error) {
 	return &msg, nil
 }
 
-func (r *RDB) dequeueSingle(queue string) (data string, err error) {
-	// timeout needed to avoid blocking forever
-	return r.client.BRPopLPush(queue, base.InProgressQueue, time.Second).Result()
-}
-
 // KEYS[1] -> asynq:in_progress
+// KEYS[2] -> asynq:paused
 // ARGV    -> List of queues to query in order
+//
+// dequeueCmd checks whether a queue is paused first, before
+// calling RPOPLPUSH to pop a task from the queue.
 var dequeueCmd = redis.NewScript(`
-local res
 for _, qkey in ipairs(ARGV) do
-	res = redis.call("RPOPLPUSH", qkey, KEYS[1])
+	if redis.call("SISMEMBER", KEYS[2], qkey) == 0 then
+		local res = redis.call("RPOPLPUSH", qkey, KEYS[1])
 		if res then
 			return res
 		end
 	end
-return res`)
+end
+return nil`)
 
-func (r *RDB) dequeue(queues ...string) (data string, err error) {
-	var args []interface{}
-	for _, qkey := range queues {
-		args = append(args, qkey)
-	}
-	res, err := dequeueCmd.Run(r.client, []string{base.InProgressQueue}, args...).Result()
+func (r *RDB) dequeue(qkeys ...interface{}) (data string, err error) {
+	res, err := dequeueCmd.Run(r.client,
+		[]string{base.InProgressQueue, base.PausedQueues}, qkeys...).Result()
 	if err != nil {
 		return "", err
 	}
@@ -167,7 +159,10 @@ func (r *RDB) dequeue(queues ...string) (data string, err error) {
 // ARGV[3] -> task ID
 // Note: LREM count ZERO means "remove all elements equal to val"
 var doneCmd = redis.NewScript(`
-redis.call("LREM", KEYS[1], 0, ARGV[1]) 
+local x = redis.call("LREM", KEYS[1], 0, ARGV[1]) 
+if x == 0 then
+  return redis.error_reply("NOT FOUND")
+end
 local n = redis.call("INCR", KEYS[2])
 if tonumber(n) == 1 then
 	redis.call("EXPIREAT", KEYS[2], ARGV[2])
@@ -289,7 +284,10 @@ func (r *RDB) ScheduleUnique(msg *base.TaskMessage, processAt time.Time, ttl tim
 // ARGV[3] -> retry_at UNIX timestamp
 // ARGV[4] -> stats expiration timestamp
 var retryCmd = redis.NewScript(`
-redis.call("LREM", KEYS[1], 0, ARGV[1])
+local x = redis.call("LREM", KEYS[1], 0, ARGV[1])
+if x == 0 then
+  return redis.error_reply("NOT FOUND")
+end
 redis.call("ZADD", KEYS[2], ARGV[3], ARGV[2])
 local n = redis.call("INCR", KEYS[3])
 if tonumber(n) == 1 then
@@ -340,7 +338,10 @@ const (
 // ARGV[5] -> max number of tasks in dead queue (e.g., 100)
 // ARGV[6] -> stats expiration timestamp
 var killCmd = redis.NewScript(`
-redis.call("LREM", KEYS[1], 0, ARGV[1])
+local x = redis.call("LREM", KEYS[1], 0, ARGV[1])
+if x == 0 then
+  return redis.error_reply("NOT FOUND")
+end
 redis.call("ZADD", KEYS[2], ARGV[3], ARGV[2])
 redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", ARGV[4])
 redis.call("ZREMRANGEBYRANK", KEYS[2], 0, -ARGV[5])
@@ -404,76 +405,59 @@ func (r *RDB) RequeueAll() (int64, error) {
 	return n, nil
 }
 
-// CheckAndEnqueue checks for all scheduled tasks and enqueues any tasks that
-// have to be processed.
-//
-// qnames specifies to which queues to send tasks.
-func (r *RDB) CheckAndEnqueue(qnames ...string) error {
+// CheckAndEnqueue checks for all scheduled/retry tasks and enqueues any tasks that
+// are ready to be processed.
+func (r *RDB) CheckAndEnqueue() (err error) {
 	delayed := []string{base.ScheduledQueue, base.RetryQueue}
 	for _, zset := range delayed {
-		var err error
-		if len(qnames) == 1 {
-			err = r.forwardSingle(zset, base.QueueKey(qnames[0]))
-		} else {
-			err = r.forward(zset)
-		}
+		n := 1
+		for n != 0 {
+			n, err = r.forward(zset)
 			if err != nil {
 				return err
 			}
 		}
+	}
 	return nil
 }
 
 // KEYS[1] -> source queue (e.g. scheduled or retry queue)
 // ARGV[1] -> current unix time
 // ARGV[2] -> queue prefix
+// Note: Script moves tasks up to 100 at a time to keep the runtime of script short.
 var forwardCmd = redis.NewScript(`
-local msgs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1])
+local msgs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, 100)
 for _, msg in ipairs(msgs) do
 	local decoded = cjson.decode(msg)
 	local qkey = ARGV[2] .. decoded["Queue"]
 	redis.call("LPUSH", qkey, msg)
 	redis.call("ZREM", KEYS[1], msg)
 end
-return msgs`)
+return table.getn(msgs)`)
 
-// forward moves all tasks with a score less than the current unix time
-// from the src zset.
-func (r *RDB) forward(src string) error {
+// forward moves tasks with a score less than the current unix time
+// from the src zset. It returns the number of tasks moved.
+func (r *RDB) forward(src string) (int, error) {
 	now := float64(time.Now().Unix())
-	return forwardCmd.Run(r.client,
-		[]string{src}, now, base.QueuePrefix).Err()
+	res, err := forwardCmd.Run(r.client,
+		[]string{src}, now, base.QueuePrefix).Result()
+	if err != nil {
+		return 0, err
 	}
-
-// KEYS[1] -> source queue (e.g. scheduled or retry queue)
-// KEYS[2] -> destination queue
-var forwardSingleCmd = redis.NewScript(`
-local msgs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1])
-for _, msg in ipairs(msgs) do
-	redis.call("LPUSH", KEYS[2], msg)
-	redis.call("ZREM", KEYS[1], msg)
-end
-return msgs`)
-
-// forwardSingle moves all tasks with a score less than the current unix time
-// from the src zset to dst list.
-func (r *RDB) forwardSingle(src, dst string) error {
-	now := float64(time.Now().Unix())
-	return forwardSingleCmd.Run(r.client,
-		[]string{src, dst}, now).Err()
+	return cast.ToInt(res), nil
 }
 
 // KEYS[1]  -> asynq:servers:<host:pid:sid>
 // KEYS[2]  -> asynq:servers
 // KEYS[3]  -> asynq:workers<host:pid:sid>
-// keys[4]  -> asynq:workers
+// KEYS[4]  -> asynq:workers
 // ARGV[1]  -> expiration time
 // ARGV[2]  -> TTL in seconds
-// ARGV[3]  -> process info
+// ARGV[3]  -> server info
 // ARGV[4:] -> alternate key-value pair of (worker id, worker data)
 // Note: Add key to ZSET with expiration time as score.
 // ref: https://github.com/antirez/redis/issues/135#issuecomment-2361996
-var writeProcessInfoCmd = redis.NewScript(`
+var writeServerStateCmd = redis.NewScript(`
 redis.call("SETEX", KEYS[1], ARGV[2], ARGV[3])
 redis.call("ZADD", KEYS[2], ARGV[1], KEYS[1])
 redis.call("DEL", KEYS[3])
@@ -485,26 +469,23 @@ redis.call("ZADD", KEYS[4], ARGV[1], KEYS[3])
 return redis.status_reply("OK")`)
 
 // WriteServerState writes server state data to redis with expiration set to the value ttl.
-func (r *RDB) WriteServerState(ss *base.ServerState, ttl time.Duration) error {
-	info := ss.GetInfo()
+func (r *RDB) WriteServerState(info *base.ServerInfo, workers []*base.WorkerInfo, ttl time.Duration) error {
 	bytes, err := json.Marshal(info)
 	if err != nil {
 		return err
 	}
-	var args []interface{} // args to the lua script
 	exp := time.Now().Add(ttl).UTC()
-	workers := ss.GetWorkers()
-	args = append(args, float64(exp.Unix()), ttl.Seconds(), bytes)
+	args := []interface{}{float64(exp.Unix()), ttl.Seconds(), bytes} // args to the lua script
 	for _, w := range workers {
 		bytes, err := json.Marshal(w)
 		if err != nil {
 			continue // skip bad data
 		}
-		args = append(args, w.ID.String(), bytes)
+		args = append(args, w.ID, bytes)
 	}
 	skey := base.ServerInfoKey(info.Host, info.PID, info.ServerID)
 	wkey := base.WorkersKey(info.Host, info.PID, info.ServerID)
-	return writeProcessInfoCmd.Run(r.client,
+	return writeServerStateCmd.Run(r.client,
 		[]string{skey, base.AllServers, wkey, base.AllWorkers},
 		args...).Err()
 }
@@ -521,11 +502,9 @@ redis.call("DEL", KEYS[4])
 return redis.status_reply("OK")`)
 
 // ClearServerState deletes server state data from redis.
-func (r *RDB) ClearServerState(ss *base.ServerState) error {
-	info := ss.GetInfo()
-	host, pid, id := info.Host, info.PID, info.ServerID
-	skey := base.ServerInfoKey(host, pid, id)
-	wkey := base.WorkersKey(host, pid, id)
+func (r *RDB) ClearServerState(host string, pid int, serverID string) error {
+	skey := base.ServerInfoKey(host, pid, serverID)
+	wkey := base.WorkersKey(host, pid, serverID)
 	return clearProcessInfoCmd.Run(r.client,
 		[]string{base.AllServers, skey, base.AllWorkers, wkey}).Err()
 }

internal/rdb/rdb_test.go

@@ -227,6 +227,97 @@ func TestDequeue(t *testing.T) {
 	}
 }
 
+func TestDequeueIgnoresPausedQueues(t *testing.T) {
+	r := setup(t)
+	t1 := h.NewTaskMessage("send_email", map[string]interface{}{"subject": "hello!"})
+	t2 := h.NewTaskMessage("export_csv", nil)
+
+	tests := []struct {
+		paused         []string // list of paused queues
+		enqueued       map[string][]*base.TaskMessage
+		args           []string // list of queues to query
+		want           *base.TaskMessage
+		err            error
+		wantEnqueued   map[string][]*base.TaskMessage
+		wantInProgress []*base.TaskMessage
+	}{
+		{
+			paused: []string{"default"},
+			enqueued: map[string][]*base.TaskMessage{
+				"default":  {t1},
+				"critical": {t2},
+			},
+			args: []string{"default", "critical"},
+			want: t2,
+			err:  nil,
+			wantEnqueued: map[string][]*base.TaskMessage{
+				"default":  {t1},
+				"critical": {},
+			},
+			wantInProgress: []*base.TaskMessage{t2},
+		},
+		{
+			paused: []string{"default"},
+			enqueued: map[string][]*base.TaskMessage{
+				"default": {t1},
+			},
+			args: []string{"default"},
+			want: nil,
+			err:  ErrNoProcessableTask,
+			wantEnqueued: map[string][]*base.TaskMessage{
+				"default": {t1},
+			},
+			wantInProgress: []*base.TaskMessage{},
+		},
+		{
+			paused: []string{"critical", "default"},
+			enqueued: map[string][]*base.TaskMessage{
+				"default":  {t1},
+				"critical": {t2},
+			},
+			args: []string{"default", "critical"},
+			want: nil,
+			err:  ErrNoProcessableTask,
+			wantEnqueued: map[string][]*base.TaskMessage{
+				"default":  {t1},
+				"critical": {t2},
+			},
+			wantInProgress: []*base.TaskMessage{},
+		},
+	}
+
+	for _, tc := range tests {
+		h.FlushDB(t, r.client) // clean up db before each test case
+		for _, qname := range tc.paused {
+			if err := r.Pause(qname); err != nil {
+				t.Fatal(err)
+			}
+		}
+		for queue, msgs := range tc.enqueued {
+			h.SeedEnqueuedQueue(t, r.client, msgs, queue)
+		}
+
+		got, err := r.Dequeue(tc.args...)
+		if !cmp.Equal(got, tc.want) || err != tc.err {
+			t.Errorf("Dequeue(%v) = %v, %v; want %v, %v",
+				tc.args, got, err, tc.want, tc.err)
+			continue
+		}
+
+		for queue, want := range tc.wantEnqueued {
+			gotEnqueued := h.GetEnqueuedMessages(t, r.client, queue)
+			if diff := cmp.Diff(want, gotEnqueued, h.SortMsgOpt); diff != "" {
+				t.Errorf("mismatch found in %q: (-want,+got):\n%s", base.QueueKey(queue), diff)
+			}
+		}
+
+		gotInProgress := h.GetInProgressMessages(t, r.client)
+		if diff := cmp.Diff(tc.wantInProgress, gotInProgress, h.SortMsgOpt); diff != "" {
+			t.Errorf("mismatch found in %q: (-want,+got):\n%s", base.InProgressQueue, diff)
+		}
+	}
+}
+
 func TestDone(t *testing.T) {
 	r := setup(t)
 	t1 := h.NewTaskMessage("send_email", nil)
@@ -769,7 +860,6 @@ func TestCheckAndEnqueue(t *testing.T) {
 	tests := []struct {
 		scheduled     []h.ZSetEntry
 		retry         []h.ZSetEntry
-		qnames        []string
 		wantEnqueued  map[string][]*base.TaskMessage
 		wantScheduled []*base.TaskMessage
 		wantRetry     []*base.TaskMessage
@@ -781,7 +871,6 @@ func TestCheckAndEnqueue(t *testing.T) {
 			},
 			retry: []h.ZSetEntry{
 				{Msg: t3, Score: float64(secondAgo.Unix())}},
-			qnames: []string{"default"},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"default": {t1, t2, t3},
 			},
@@ -794,7 +883,6 @@ func TestCheckAndEnqueue(t *testing.T) {
 				{Msg: t2, Score: float64(secondAgo.Unix())}},
 			retry: []h.ZSetEntry{
 				{Msg: t3, Score: float64(secondAgo.Unix())}},
-			qnames: []string{"default"},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"default": {t2, t3},
 			},
@@ -807,7 +895,6 @@ func TestCheckAndEnqueue(t *testing.T) {
 				{Msg: t2, Score: float64(hourFromNow.Unix())}},
 			retry: []h.ZSetEntry{
 				{Msg: t3, Score: float64(hourFromNow.Unix())}},
-			qnames: []string{"default"},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"default": {},
 			},
@@ -821,7 +908,6 @@ func TestCheckAndEnqueue(t *testing.T) {
 			},
 			retry: []h.ZSetEntry{
 				{Msg: t5, Score: float64(secondAgo.Unix())}},
-			qnames: []string{"default", "critical", "low"},
 			wantEnqueued: map[string][]*base.TaskMessage{
 				"default":  {t1},
 				"critical": {t4},
@@ -837,7 +923,7 @@ func TestCheckAndEnqueue(t *testing.T) {
 		h.SeedScheduledQueue(t, r.client, tc.scheduled)
 		h.SeedRetryQueue(t, r.client, tc.retry)
 
-		err := r.CheckAndEnqueue(tc.qnames...)
+		err := r.CheckAndEnqueue()
 		if err != nil {
 			t.Errorf("(*RDB).CheckScheduled() = %v, want nil", err)
 			continue
@@ -864,64 +950,63 @@ func TestCheckAndEnqueue(t *testing.T) {
 
 func TestWriteServerState(t *testing.T) {
 	r := setup(t)
-	queues := map[string]int{"default": 2, "email": 5, "low": 1}
 
-	started := time.Now()
-	ss := base.NewServerState("localhost", 4242, 10, queues, false)
-	ss.SetStarted(started)
-	ss.SetStatus(base.StatusRunning)
-	ttl := 5 * time.Second
+	var (
+		host     = "localhost"
+		pid      = 4242
+		serverID = "server123"
 
-	h.FlushDB(t, r.client)
+		ttl = 5 * time.Second
+	)
 
-	err := r.WriteServerState(ss, ttl)
+	info := base.ServerInfo{
+		Host:              host,
+		PID:               pid,
+		ServerID:          serverID,
+		Concurrency:       10,
+		Queues:            map[string]int{"default": 2, "email": 5, "low": 1},
+		StrictPriority:    false,
+		Started:           time.Now(),
+		Status:            "running",
+		ActiveWorkerCount: 0,
+	}
+
+	err := r.WriteServerState(&info, nil /* workers */, ttl)
 	if err != nil {
 		t.Errorf("r.WriteServerState returned an error: %v", err)
 	}
 
-	// Check ServerInfo was written correctly
-	info := ss.GetInfo()
-	skey := base.ServerInfoKey(info.Host, info.PID, info.ServerID)
+	// Check ServerInfo was written correctly.
+	skey := base.ServerInfoKey(host, pid, serverID)
 	data := r.client.Get(skey).Val()
 	var got base.ServerInfo
 	err = json.Unmarshal([]byte(data), &got)
 	if err != nil {
 		t.Fatalf("could not decode json: %v", err)
 	}
-	want := base.ServerInfo{
-		Host:              info.Host,
-		PID:               info.PID,
-		Concurrency:       info.Concurrency,
-		Queues:            map[string]int{"default": 2, "email": 5, "low": 1},
-		StrictPriority:    false,
-		Status:            "running",
-		Started:           started,
-		ActiveWorkerCount: 0,
-	}
-	ignoreOpt := cmpopts.IgnoreFields(base.ServerInfo{}, "ServerID")
-	if diff := cmp.Diff(want, got, ignoreOpt); diff != "" {
+	if diff := cmp.Diff(info, got); diff != "" {
 		t.Errorf("persisted ServerInfo was %v, want %v; (-want,+got)\n%s",
-			got, want, diff)
+			got, info, diff)
 	}
-	// Check ServerInfo TTL was set correctly
+	// Check ServerInfo TTL was set correctly.
 	gotTTL := r.client.TTL(skey).Val()
 	if !cmp.Equal(ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) {
 		t.Errorf("TTL of %q was %v, want %v", skey, gotTTL, ttl)
 	}
-	// Check ServerInfo key was added to the set correctly
-	gotProcesses := r.client.ZRange(base.AllServers, 0, -1).Val()
-	wantProcesses := []string{skey}
-	if diff := cmp.Diff(wantProcesses, gotProcesses); diff != "" {
-		t.Errorf("%q contained %v, want %v", base.AllServers, gotProcesses, wantProcesses)
+	// Check ServerInfo key was added to the set all server keys correctly.
+	gotServerKeys := r.client.ZRange(base.AllServers, 0, -1).Val()
+	wantServerKeys := []string{skey}
+	if diff := cmp.Diff(wantServerKeys, gotServerKeys); diff != "" {
+		t.Errorf("%q contained %v, want %v", base.AllServers, gotServerKeys, wantServerKeys)
 	}
 
-	// Check WorkersInfo was written correctly
-	wkey := base.WorkersKey(info.Host, info.PID, info.ServerID)
+	// Check WorkersInfo was written correctly.
+	wkey := base.WorkersKey(host, pid, serverID)
 	workerExist := r.client.Exists(wkey).Val()
 	if workerExist != 0 {
 		t.Errorf("%q key exists", wkey)
 	}
-	// Check WorkersInfo key was added to the set correctly
+	// Check WorkersInfo key was added to the set correctly.
 	gotWorkerKeys := r.client.ZRange(base.AllWorkers, 0, -1).Val()
 	wantWorkerKeys := []string{wkey}
 	if diff := cmp.Diff(wantWorkerKeys, gotWorkerKeys); diff != "" {
@@ -931,109 +1016,105 @@ func TestWriteServerState(t *testing.T) {
 
 func TestWriteServerStateWithWorkers(t *testing.T) {
 	r := setup(t)
-	queues := map[string]int{"default": 2, "email": 5, "low": 1}
-	concurrency := 10
 
-	started := time.Now().Add(-10 * time.Minute)
-	w1Started := time.Now().Add(-time.Minute)
-	w2Started := time.Now().Add(-time.Second)
-	msg1 := h.NewTaskMessage("send_email", map[string]interface{}{"user_id": "123"})
-	msg2 := h.NewTaskMessage("gen_thumbnail", map[string]interface{}{"path": "some/path/to/imgfile"})
-	ss := base.NewServerState("127.0.01", 4242, concurrency, queues, false)
-	ss.SetStarted(started)
-	ss.SetStatus(base.StatusRunning)
-	ss.AddWorkerStats(msg1, w1Started)
-	ss.AddWorkerStats(msg2, w2Started)
-	ttl := 5 * time.Second
+	var (
+		host     = "127.0.0.1"
+		pid      = 4242
+		serverID = "server123"
 
-	h.FlushDB(t, r.client)
+		msg1 = h.NewTaskMessage("send_email", map[string]interface{}{"user_id": "123"})
+		msg2 = h.NewTaskMessage("gen_thumbnail", map[string]interface{}{"path": "some/path/to/imgfile"})
 
-	err := r.WriteServerState(ss, ttl)
-	if err != nil {
-		t.Errorf("r.WriteServerState returned an error: %v", err)
+		ttl = 5 * time.Second
+	)
+
+	workers := []*base.WorkerInfo{
+		{
+			Host:    host,
+			PID:     pid,
+			ID:      msg1.ID.String(),
+			Type:    msg1.Type,
+			Queue:   msg1.Queue,
+			Payload: msg1.Payload,
+			Started: time.Now().Add(-10 * time.Second),
+		},
+		{
+			Host:    host,
+			PID:     pid,
+			ID:      msg2.ID.String(),
+			Type:    msg2.Type,
+			Queue:   msg2.Queue,
+			Payload: msg2.Payload,
+			Started: time.Now().Add(-2 * time.Minute),
+		},
 	}
 
-	// Check ServerInfo was written correctly
-	info := ss.GetInfo()
-	skey := base.ServerInfoKey(info.Host, info.PID, info.ServerID)
+	serverInfo := base.ServerInfo{
+		Host:              host,
+		PID:               pid,
+		ServerID:          serverID,
+		Concurrency:       10,
+		Queues:            map[string]int{"default": 2, "email": 5, "low": 1},
+		StrictPriority:    false,
+		Started:           time.Now().Add(-10 * time.Minute),
+		Status:            "running",
+		ActiveWorkerCount: len(workers),
+	}
+
+	err := r.WriteServerState(&serverInfo, workers, ttl)
+	if err != nil {
+		t.Fatalf("r.WriteServerState returned an error: %v", err)
+	}
+
+	// Check ServerInfo was written correctly.
+	skey := base.ServerInfoKey(host, pid, serverID)
 	data := r.client.Get(skey).Val()
 	var got base.ServerInfo
 	err = json.Unmarshal([]byte(data), &got)
 	if err != nil {
 		t.Fatalf("could not decode json: %v", err)
 	}
-	want := base.ServerInfo{
-		Host:              info.Host,
-		PID:               info.PID,
-		ServerID:          info.ServerID,
-		Concurrency:       concurrency,
-		Queues:            queues,
-		StrictPriority:    false,
-		Status:            "running",
-		Started:           started,
-		ActiveWorkerCount: 2,
-	}
-	if diff := cmp.Diff(want, got); diff != "" {
+	if diff := cmp.Diff(serverInfo, got); diff != "" {
 		t.Errorf("persisted ServerInfo was %v, want %v; (-want,+got)\n%s",
-			got, want, diff)
+			got, serverInfo, diff)
 	}
-	// Check ServerInfo TTL was set correctly
+	// Check ServerInfo TTL was set correctly.
 	gotTTL := r.client.TTL(skey).Val()
 	if !cmp.Equal(ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) {
 		t.Errorf("TTL of %q was %v, want %v", skey, gotTTL, ttl)
 	}
-	// Check ServerInfo key was added to the set correctly
-	gotProcesses := r.client.ZRange(base.AllServers, 0, -1).Val()
-	wantProcesses := []string{skey}
-	if diff := cmp.Diff(wantProcesses, gotProcesses); diff != "" {
-		t.Errorf("%q contained %v, want %v", base.AllServers, gotProcesses, wantProcesses)
+	// Check ServerInfo key was added to the set correctly.
+	gotServerKeys := r.client.ZRange(base.AllServers, 0, -1).Val()
+	wantServerKeys := []string{skey}
+	if diff := cmp.Diff(wantServerKeys, gotServerKeys); diff != "" {
+		t.Errorf("%q contained %v, want %v", base.AllServers, gotServerKeys, wantServerKeys)
 	}
 
-	// Check WorkersInfo was written correctly
-	wkey := base.WorkersKey(info.Host, info.PID, info.ServerID)
+	// Check WorkersInfo was written correctly.
+	wkey := base.WorkersKey(host, pid, serverID)
 	wdata := r.client.HGetAll(wkey).Val()
 	if len(wdata) != 2 {
 		t.Fatalf("HGETALL %q returned a hash of size %d, want 2", wkey, len(wdata))
 	}
-	gotWorkers := make(map[string]*base.WorkerInfo)
-	for key, val := range wdata {
+	var gotWorkers []*base.WorkerInfo
+	for _, val := range wdata {
 		var w base.WorkerInfo
 		if err := json.Unmarshal([]byte(val), &w); err != nil {
 			t.Fatalf("could not unmarshal worker's data: %v", err)
 		}
-		gotWorkers[key] = &w
+		gotWorkers = append(gotWorkers, &w)
 	}
-	wantWorkers := map[string]*base.WorkerInfo{
-		msg1.ID.String(): {
-			Host:    info.Host,
-			PID:     info.PID,
-			ID:      msg1.ID,
-			Type:    msg1.Type,
-			Queue:   msg1.Queue,
-			Payload: msg1.Payload,
-			Started: w1Started,
-		},
-		msg2.ID.String(): {
-			Host:    info.Host,
-			PID:     info.PID,
-			ID:      msg2.ID,
-			Type:    msg2.Type,
-			Queue:   msg2.Queue,
-			Payload: msg2.Payload,
-			Started: w2Started,
-		},
-	}
-	if diff := cmp.Diff(wantWorkers, gotWorkers); diff != "" {
+	if diff := cmp.Diff(workers, gotWorkers, h.SortWorkerInfoOpt); diff != "" {
 		t.Errorf("persisted workers info was %v, want %v; (-want,+got)\n%s",
-			gotWorkers, wantWorkers, diff)
+			gotWorkers, workers, diff)
 	}
 
-	// Check WorkersInfo TTL was set correctly
+	// Check WorkersInfo TTL was set correctly.
 	gotTTL = r.client.TTL(wkey).Val()
-	if !cmp.Equal(ttl, gotTTL, timeCmpOpt) {
+	if !cmp.Equal(ttl.Seconds(), gotTTL.Seconds(), cmpopts.EquateApprox(0, 1)) {
 		t.Errorf("TTL of %q was %v, want %v", wkey, gotTTL, ttl)
 	}
-	// Check WorkersInfo key was added to the set correctly
+	// Check WorkersInfo key was added to the set correctly.
 	gotWorkerKeys := r.client.ZRange(base.AllWorkers, 0, -1).Val()
 	wantWorkerKeys := []string{wkey}
 	if diff := cmp.Diff(wantWorkerKeys, gotWorkerKeys); diff != "" {
@@ -1043,51 +1124,96 @@ func TestWriteServerStateWithWorkers(t *testing.T) {
 
 func TestClearServerState(t *testing.T) {
 	r := setup(t)
-	ss := base.NewServerState("127.0.01", 4242, 10, map[string]int{"default": 1}, false)
-	info := ss.GetInfo()
 
-	h.FlushDB(t, r.client)
+	var (
+		host     = "127.0.0.1"
+		pid      = 1234
+		serverID = "server123"
 
-	skey := base.ServerInfoKey(info.Host, info.PID, info.ServerID)
-	wkey := base.WorkersKey(info.Host, info.PID, info.ServerID)
-	otherSKey := base.ServerInfoKey("otherhost", 12345, "server98")
-	otherWKey := base.WorkersKey("otherhost", 12345, "server98")
-	// Populate the keys.
-	if err := r.client.Set(skey, "process-info", 0).Err(); err != nil {
-		t.Fatal(err)
+		otherHost     = "127.0.0.2"
+		otherPID      = 9876
+		otherServerID = "server987"
+
+		msg1 = h.NewTaskMessage("send_email", map[string]interface{}{"user_id": "123"})
+		msg2 = h.NewTaskMessage("gen_thumbnail", map[string]interface{}{"path": "some/path/to/imgfile"})
+
+		ttl = 5 * time.Second
+	)
+
+	workers1 := []*base.WorkerInfo{
+		{
+			Host:    host,
+			PID:     pid,
+			ID:      msg1.ID.String(),
+			Type:    msg1.Type,
+			Queue:   msg1.Queue,
+			Payload: msg1.Payload,
+			Started: time.Now().Add(-10 * time.Second),
+		},
 	}
-	if err := r.client.HSet(wkey, "worker-key", "worker-info").Err(); err != nil {
-		t.Fatal(err)
-	}
-	if err := r.client.ZAdd(base.AllServers, &redis.Z{Member: skey}).Err(); err != nil {
-		t.Fatal(err)
-	}
-	if err := r.client.ZAdd(base.AllServers, &redis.Z{Member: otherSKey}).Err(); err != nil {
-		t.Fatal(err)
-	}
-	if err := r.client.ZAdd(base.AllWorkers, &redis.Z{Member: wkey}).Err(); err != nil {
-		t.Fatal(err)
-	}
-	if err := r.client.ZAdd(base.AllWorkers, &redis.Z{Member: otherWKey}).Err(); err != nil {
-		t.Fatal(err)
+	serverInfo1 := base.ServerInfo{
+		Host:              host,
+		PID:               pid,
+		ServerID:          serverID,
+		Concurrency:       10,
+		Queues:            map[string]int{"default": 2, "email": 5, "low": 1},
+		StrictPriority:    false,
+		Started:           time.Now().Add(-10 * time.Minute),
+		Status:            "running",
+		ActiveWorkerCount: len(workers1),
 	}
 
-	err := r.ClearServerState(ss)
+	workers2 := []*base.WorkerInfo{
+		{
+			Host:    otherHost,
+			PID:     otherPID,
+			ID:      msg2.ID.String(),
+			Type:    msg2.Type,
+			Queue:   msg2.Queue,
+			Payload: msg2.Payload,
+			Started: time.Now().Add(-30 * time.Second),
+		},
+	}
+	serverInfo2 := base.ServerInfo{
+		Host:              otherHost,
+		PID:               otherPID,
+		ServerID:          otherServerID,
+		Concurrency:       10,
+		Queues:            map[string]int{"default": 2, "email": 5, "low": 1},
+		StrictPriority:    false,
+		Started:           time.Now().Add(-15 * time.Minute),
+		Status:            "running",
+		ActiveWorkerCount: len(workers2),
+	}
+
+	// Write server and workers data.
+	if err := r.WriteServerState(&serverInfo1, workers1, ttl); err != nil {
+		t.Fatalf("could not write server state: %v", err)
+	}
+	if err := r.WriteServerState(&serverInfo2, workers2, ttl); err != nil {
+		t.Fatalf("could not write server state: %v", err)
+	}
+
+	err := r.ClearServerState(host, pid, serverID)
 	if err != nil {
 		t.Fatalf("(*RDB).ClearServerState failed: %v", err)
 	}
 
-	// Check all keys are cleared
+	skey := base.ServerInfoKey(host, pid, serverID)
+	wkey := base.WorkersKey(host, pid, serverID)
+	otherSKey := base.ServerInfoKey(otherHost, otherPID, otherServerID)
+	otherWKey := base.WorkersKey(otherHost, otherPID, otherServerID)
+	// Check all keys are cleared.
 	if r.client.Exists(skey).Val() != 0 {
 		t.Errorf("Redis key %q exists", skey)
 	}
 	if r.client.Exists(wkey).Val() != 0 {
 		t.Errorf("Redis key %q exists", wkey)
 	}
-	gotProcessKeys := r.client.ZRange(base.AllServers, 0, -1).Val()
-	wantProcessKeys := []string{otherSKey}
-	if diff := cmp.Diff(wantProcessKeys, gotProcessKeys); diff != "" {
-		t.Errorf("%q contained %v, want %v", base.AllServers, gotProcessKeys, wantProcessKeys)
+	gotServerKeys := r.client.ZRange(base.AllServers, 0, -1).Val()
+	wantServerKeys := []string{otherSKey}
+	if diff := cmp.Diff(wantServerKeys, gotServerKeys); diff != "" {
+		t.Errorf("%q contained %v, want %v", base.AllServers, gotServerKeys, wantServerKeys)
 	}
 	gotWorkerKeys := r.client.ZRange(base.AllWorkers, 0, -1).Val()
 	wantWorkerKeys := []string{otherWKey}

internal/testbroker/testbroker.go

@@ -132,7 +132,7 @@ func (tb *TestBroker) RequeueAll() (int64, error) {
 	return tb.real.RequeueAll()
 }
 
-func (tb *TestBroker) CheckAndEnqueue(qnames ...string) error {
+func (tb *TestBroker) CheckAndEnqueue() error {
 	tb.mu.Lock()
 	defer tb.mu.Unlock()
 	if tb.sleeping {
@@ -141,22 +141,22 @@ func (tb *TestBroker) CheckAndEnqueue(qnames ...string) error {
 	return tb.real.CheckAndEnqueue()
 }
 
-func (tb *TestBroker) WriteServerState(ss *base.ServerState, ttl time.Duration) error {
+func (tb *TestBroker) WriteServerState(info *base.ServerInfo, workers []*base.WorkerInfo, ttl time.Duration) error {
 	tb.mu.Lock()
 	defer tb.mu.Unlock()
 	if tb.sleeping {
 		return errRedisDown
 	}
-	return tb.real.WriteServerState(ss, ttl)
+	return tb.real.WriteServerState(info, workers, ttl)
 }
 
-func (tb *TestBroker) ClearServerState(ss *base.ServerState) error {
+func (tb *TestBroker) ClearServerState(host string, pid int, serverID string) error {
 	tb.mu.Lock()
 	defer tb.mu.Unlock()
 	if tb.sleeping {
 		return errRedisDown
 	}
-	return tb.real.ClearServerState(ss)
+	return tb.real.ClearServerState(host, pid, serverID)
 }
 
 func (tb *TestBroker) CancelationPubSub() (*redis.PubSub, error) {

processor.go

@@ -13,16 +13,15 @@ import (
 	"time"
 
 	"github.com/hibiken/asynq/internal/base"
+	"github.com/hibiken/asynq/internal/log"
 	"github.com/hibiken/asynq/internal/rdb"
 	"golang.org/x/time/rate"
 )
 
 type processor struct {
-	logger Logger
+	logger *log.Logger
 	broker base.Broker
 
-	ss *base.ServerState
-
 	handler Handler
 
 	queueConfig map[string]int
@@ -59,45 +58,52 @@ type processor struct {
 
 	// cancelations is a set of cancel functions for all in-progress tasks.
 	cancelations *base.Cancelations
+
+	starting chan<- *base.TaskMessage
+	finished chan<- *base.TaskMessage
 }
 
 type retryDelayFunc func(n int, err error, task *Task) time.Duration
 
-type newProcessorParams struct {
-	logger          Logger
+type processorParams struct {
+	logger          *log.Logger
 	broker          base.Broker
-	ss              *base.ServerState
 	retryDelayFunc  retryDelayFunc
 	syncCh          chan<- *syncRequest
 	cancelations    *base.Cancelations
+	concurrency     int
+	queues          map[string]int
+	strictPriority  bool
 	errHandler      ErrorHandler
 	shutdownTimeout time.Duration
+	starting        chan<- *base.TaskMessage
+	finished        chan<- *base.TaskMessage
 }
 
 // newProcessor constructs a new processor.
-func newProcessor(params newProcessorParams) *processor {
-	info := params.ss.GetInfo()
-	qcfg := normalizeQueueCfg(info.Queues)
+func newProcessor(params processorParams) *processor {
+	queues := normalizeQueues(params.queues)
 	orderedQueues := []string(nil)
-	if info.StrictPriority {
-		orderedQueues = sortByPriority(qcfg)
+	if params.strictPriority {
+		orderedQueues = sortByPriority(queues)
 	}
 	return &processor{
 		logger:         params.logger,
 		broker:         params.broker,
-		ss:             params.ss,
-		queueConfig:    qcfg,
+		queueConfig:    queues,
 		orderedQueues:  orderedQueues,
 		retryDelayFunc: params.retryDelayFunc,
 		syncRequestCh:  params.syncCh,
 		cancelations:   params.cancelations,
 		errLogLimiter:  rate.NewLimiter(rate.Every(3*time.Second), 1),
-		sema:           make(chan struct{}, info.Concurrency),
+		sema:           make(chan struct{}, params.concurrency),
 		done:           make(chan struct{}),
 		abort:          make(chan struct{}),
 		quit:           make(chan struct{}),
 		errHandler:     params.errHandler,
 		handler:        HandlerFunc(func(ctx context.Context, t *Task) error { return fmt.Errorf("handler not set") }),
+		starting:       params.starting,
+		finished:       params.finished,
 	}
 }
 
@@ -105,7 +111,7 @@ func newProcessor(params newProcessorParams) *processor {
 // It's safe to call this method multiple times.
 func (p *processor) stop() {
 	p.once.Do(func() {
-		p.logger.Info("Processor shutting down...")
+		p.logger.Debug("Processor shutting down...")
 		// Unblock if processor is waiting for sema token.
 		close(p.abort)
 		// Signal the processor goroutine to stop processing tasks
@@ -144,7 +150,7 @@ func (p *processor) start(wg *sync.WaitGroup) {
 		for {
 			select {
 			case <-p.done:
-				p.logger.Info("Processor done")
+				p.logger.Debug("Processor done")
 				return
 			default:
 				p.exec()
@@ -158,19 +164,18 @@ func (p *processor) start(wg *sync.WaitGroup) {
 func (p *processor) exec() {
 	qnames := p.queues()
 	msg, err := p.broker.Dequeue(qnames...)
-	if err == rdb.ErrNoProcessableTask { // TODO: Need to decouple this error from rdb to support other brokers
-		// queues are empty, this is a normal behavior.
-		if len(p.queueConfig) > 1 {
-			// sleep to avoid slamming redis and let scheduler move tasks into queues.
-			// Note: With multiple queues, we are not using blocking pop operation and
-			// polling queues instead. This adds significant load to redis.
+	switch {
+	case err == rdb.ErrNoProcessableTask:
+		p.logger.Debug("All queues are empty")
+		// Queues are empty, this is a normal behavior.
+		// Sleep to avoid slamming redis and let scheduler move tasks into queues.
+		// Note: We are not using blocking pop operation and polling queues instead.
+		// This adds significant load to redis.
 		time.Sleep(time.Second)
-		}
 		return
-	}
-	if err != nil {
+	case err != nil:
 		if p.errLogLimiter.Allow() {
-			p.logger.Error("Dequeue error: %v", err)
+			p.logger.Errorf("Dequeue error: %v", err)
 		}
 		return
 	}
@@ -181,26 +186,28 @@ func (p *processor) exec() {
 		p.requeue(msg)
 		return
 	case p.sema <- struct{}{}: // acquire token
-		p.ss.AddWorkerStats(msg, time.Now())
+		p.starting <- msg
 		go func() {
 			defer func() {
-				p.ss.DeleteWorkerStats(msg)
-				<-p.sema /* release token */
+				p.finished <- msg
+				<-p.sema // release token
+			}()
+
+			ctx, cancel := createContext(msg)
+			p.cancelations.Add(msg.ID.String(), cancel)
+			defer func() {
+				cancel()
+				p.cancelations.Delete(msg.ID.String())
 			}()
 
 			resCh := make(chan error, 1)
 			task := NewTask(msg.Type, msg.Payload)
-			ctx, cancel := createContext(msg)
-			p.cancelations.Add(msg.ID.String(), cancel)
-			go func() {
-				resCh <- perform(ctx, task, p.handler)
-				p.cancelations.Delete(msg.ID.String())
-			}()
+			go func() { resCh <- perform(ctx, task, p.handler) }()
 
 			select {
 			case <-p.quit:
 				// time is up, quit this worker goroutine.
-				p.logger.Warn("Quitting worker. task id=%s", msg.ID)
+				p.logger.Warnf("Quitting worker. task id=%s", msg.ID)
 				return
 			case resErr := <-resCh:
 				// Note: One of three things should happen.
@@ -229,25 +236,25 @@ func (p *processor) exec() {
 func (p *processor) restore() {
 	n, err := p.broker.RequeueAll()
 	if err != nil {
-		p.logger.Error("Could not restore unfinished tasks: %v", err)
+		p.logger.Errorf("Could not restore unfinished tasks: %v", err)
 	}
 	if n > 0 {
-		p.logger.Info("Restored %d unfinished tasks back to queue", n)
+		p.logger.Infof("Restored %d unfinished tasks back to queue", n)
 	}
 }
 
 func (p *processor) requeue(msg *base.TaskMessage) {
 	err := p.broker.Requeue(msg)
 	if err != nil {
-		p.logger.Error("Could not push task id=%s back to queue: %v", msg.ID, err)
+		p.logger.Errorf("Could not push task id=%s back to queue: %v", msg.ID, err)
 	}
 }
 
 func (p *processor) markAsDone(msg *base.TaskMessage) {
 	err := p.broker.Done(msg)
 	if err != nil {
-		errMsg := fmt.Sprintf("Could not remove task id=%s from %q", msg.ID, base.InProgressQueue)
-		p.logger.Warn("%s; Will retry syncing", errMsg)
+		errMsg := fmt.Sprintf("Could not remove task id=%s type=%q from %q", msg.ID, msg.Type, base.InProgressQueue)
+		p.logger.Warnf("%s; Will retry syncing", errMsg)
 		p.syncRequestCh <- &syncRequest{
 			fn: func() error {
 				return p.broker.Done(msg)
@@ -263,7 +270,7 @@ func (p *processor) retry(msg *base.TaskMessage, e error) {
 	err := p.broker.Retry(msg, retryAt, e.Error())
 	if err != nil {
 		errMsg := fmt.Sprintf("Could not move task id=%s from %q to %q", msg.ID, base.InProgressQueue, base.RetryQueue)
-		p.logger.Warn("%s; Will retry syncing", errMsg)
+		p.logger.Warnf("%s; Will retry syncing", errMsg)
 		p.syncRequestCh <- &syncRequest{
 			fn: func() error {
 				return p.broker.Retry(msg, retryAt, e.Error())
@@ -274,11 +281,11 @@ func (p *processor) retry(msg *base.TaskMessage, e error) {
 }
 
 func (p *processor) kill(msg *base.TaskMessage, e error) {
-	p.logger.Warn("Retry exhausted for task id=%s", msg.ID)
+	p.logger.Warnf("Retry exhausted for task id=%s", msg.ID)
 	err := p.broker.Kill(msg, e.Error())
 	if err != nil {
 		errMsg := fmt.Sprintf("Could not move task id=%s from %q to %q", msg.ID, base.InProgressQueue, base.DeadQueue)
-		p.logger.Warn("%s; Will retry syncing", errMsg)
+		p.logger.Warnf("%s; Will retry syncing", errMsg)
 		p.syncRequestCh <- &syncRequest{
 			fn: func() error {
 				return p.broker.Kill(msg, e.Error())
@@ -306,7 +313,7 @@ func (p *processor) queues() []string {
 	}
 	var names []string
 	for qname, priority := range p.queueConfig {
-		for i := 0; i < int(priority); i++ {
+		for i := 0; i < priority; i++ {
 			names = append(names, qname)
 		}
 	}
@@ -370,16 +377,15 @@ func (x byPriority) Len() int           { return len(x) }
 func (x byPriority) Less(i, j int) bool { return x[i].priority < x[j].priority }
 func (x byPriority) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
 
-// normalizeQueueCfg divides priority numbers by their
-// greatest common divisor.
-func normalizeQueueCfg(queueCfg map[string]int) map[string]int {
+// normalizeQueues divides priority numbers by their greatest common divisor.
+func normalizeQueues(queues map[string]int) map[string]int {
 	var xs []int
-	for _, x := range queueCfg {
+	for _, x := range queues {
 		xs = append(xs, x)
 	}
 	d := gcd(xs...)
 	res := make(map[string]int)
-	for q, x := range queueCfg {
+	for q, x := range queues {
 		res[q] = x / d
 	}
 	return res
@@ -401,20 +407,3 @@ func gcd(xs ...int) int {
 	}
 	return res
 }
-
-// createContext returns a context and cancel function for a given task message.
-func createContext(msg *base.TaskMessage) (ctx context.Context, cancel context.CancelFunc) {
-	ctx = context.Background()
-	timeout, err := time.ParseDuration(msg.Timeout)
-	if err == nil && timeout != 0 {
-		ctx, cancel = context.WithTimeout(ctx, timeout)
-	}
-	deadline, err := time.Parse(time.RFC3339, msg.Deadline)
-	if err == nil && !deadline.IsZero() {
-		ctx, cancel = context.WithDeadline(ctx, deadline)
-	}
-	if cancel == nil {
-		ctx, cancel = context.WithCancel(ctx)
-	}
-	return ctx, cancel
-}

processor_test.go

@@ -17,9 +17,20 @@ import (
 	h "github.com/hibiken/asynq/internal/asynqtest"
 	"github.com/hibiken/asynq/internal/base"
 	"github.com/hibiken/asynq/internal/rdb"
-	"github.com/rs/xid"
 )
 
+// fakeHeartbeater receives from starting and finished channels and do nothing.
+func fakeHeartbeater(starting, finished <-chan *base.TaskMessage, done <-chan struct{}) {
+	for {
+		select {
+		case <-starting:
+		case <-finished:
+		case <-done:
+			return
+		}
+	}
+}
+
 func TestProcessorSuccess(t *testing.T) {
 	r := setup(t)
 	rdbClient := rdb.NewRDB(r)
@@ -37,19 +48,16 @@ func TestProcessorSuccess(t *testing.T) {
 	tests := []struct {
 		enqueued      []*base.TaskMessage // initial default queue state
 		incoming      []*base.TaskMessage // tasks to be enqueued during run
-		wait          time.Duration       // wait duration between starting and stopping processor for this test case
 		wantProcessed []*Task             // tasks to be processed at the end
 	}{
 		{
 			enqueued:      []*base.TaskMessage{m1},
 			incoming:      []*base.TaskMessage{m2, m3, m4},
-			wait:          time.Second,
 			wantProcessed: []*Task{t1, t2, t3, t4},
 		},
 		{
 			enqueued:      []*base.TaskMessage{},
 			incoming:      []*base.TaskMessage{m1},
-			wait:          time.Second,
 			wantProcessed: []*Task{t1},
 		},
 	}
@@ -67,22 +75,28 @@ func TestProcessorSuccess(t *testing.T) {
 			processed = append(processed, task)
 			return nil
 		}
-		ss := base.NewServerState("localhost", 1234, 10, defaultQueueConfig, false)
-		cancelations := base.NewCancelations()
-		p := newProcessor(newProcessorParams{
+		starting := make(chan *base.TaskMessage)
+		finished := make(chan *base.TaskMessage)
+		done := make(chan struct{})
+		defer func() { close(done) }()
+		go fakeHeartbeater(starting, finished, done)
+		p := newProcessor(processorParams{
 			logger:          testLogger,
 			broker:          rdbClient,
-			ss:              ss,
 			retryDelayFunc:  defaultDelayFunc,
 			syncCh:          nil,
-			cancelations:    cancelations,
+			cancelations:    base.NewCancelations(),
+			concurrency:     10,
+			queues:          defaultQueueConfig,
+			strictPriority:  false,
 			errHandler:      nil,
 			shutdownTimeout: defaultShutdownTimeout,
+			starting:        starting,
+			finished:        finished,
 		})
 		p.handler = HandlerFunc(handler)
 
-		var wg sync.WaitGroup
-		p.start(&wg)
+		p.start(&sync.WaitGroup{})
 		for _, msg := range tc.incoming {
 			err := rdbClient.Enqueue(msg)
 			if err != nil {
@@ -90,12 +104,14 @@ func TestProcessorSuccess(t *testing.T) {
 				t.Fatal(err)
 			}
 		}
-		time.Sleep(tc.wait)
+		time.Sleep(2 * time.Second) // wait for two second to allow all enqueued tasks to be processed.
 		p.terminate()
 
+		mu.Lock()
 		if diff := cmp.Diff(tc.wantProcessed, processed, sortTaskOpt, cmp.AllowUnexported(Payload{})); diff != "" {
 			t.Errorf("mismatch found in processed tasks; (-want, +got)\n%s", diff)
 		}
+		mu.Unlock()
 
 		if l := r.LLen(base.InProgressQueue).Val(); l != 0 {
 			t.Errorf("%q has %d tasks, want 0", base.InProgressQueue, l)
@@ -146,7 +162,7 @@ func TestProcessorRetry(t *testing.T) {
 			handler: HandlerFunc(func(ctx context.Context, task *Task) error {
 				return fmt.Errorf(errMsg)
 			}),
-			wait: time.Second,
+			wait: 2 * time.Second,
 			wantRetry: []h.ZSetEntry{
 				{Msg: &r2, Score: float64(now.Add(time.Minute).Unix())},
 				{Msg: &r3, Score: float64(now.Add(time.Minute).Unix())},
@@ -174,22 +190,28 @@ func TestProcessorRetry(t *testing.T) {
 			defer mu.Unlock()
 			n++
 		}
-		ss := base.NewServerState("localhost", 1234, 10, defaultQueueConfig, false)
-		cancelations := base.NewCancelations()
-		p := newProcessor(newProcessorParams{
+		starting := make(chan *base.TaskMessage)
+		finished := make(chan *base.TaskMessage)
+		done := make(chan struct{})
+		defer func() { close(done) }()
+		go fakeHeartbeater(starting, finished, done)
+		p := newProcessor(processorParams{
 			logger:          testLogger,
 			broker:          rdbClient,
-			ss:              ss,
 			retryDelayFunc:  delayFunc,
 			syncCh:          nil,
-			cancelations:    cancelations,
+			cancelations:    base.NewCancelations(),
+			concurrency:     10,
+			queues:          defaultQueueConfig,
+			strictPriority:  false,
 			errHandler:      ErrorHandlerFunc(errHandler),
 			shutdownTimeout: defaultShutdownTimeout,
+			starting:        starting,
+			finished:        finished,
 		})
 		p.handler = tc.handler
 
-		var wg sync.WaitGroup
-		p.start(&wg)
+		p.start(&sync.WaitGroup{})
 		for _, msg := range tc.incoming {
 			err := rdbClient.Enqueue(msg)
 			if err != nil {
@@ -197,10 +219,10 @@ func TestProcessorRetry(t *testing.T) {
 				t.Fatal(err)
 			}
 		}
-		time.Sleep(tc.wait)
+		time.Sleep(tc.wait) // FIXME: This makes test flaky.
 		p.terminate()
 
-		cmpOpt := cmpopts.EquateApprox(0, float64(time.Second)) // allow up to second difference in zset score
+		cmpOpt := cmpopts.EquateApprox(0, float64(time.Second)) // allow up to a second difference in zset score
 		gotRetry := h.GetRetryEntries(t, r)
 		if diff := cmp.Diff(tc.wantRetry, gotRetry, h.SortZSetEntryOpt, cmpOpt); diff != "" {
 			t.Errorf("mismatch found in %q after running processor; (-want, +got)\n%s", base.RetryQueue, diff)
@@ -249,17 +271,24 @@ func TestProcessorQueues(t *testing.T) {
 	}
 
 	for _, tc := range tests {
-		cancelations := base.NewCancelations()
-		ss := base.NewServerState("localhost", 1234, 10, tc.queueCfg, false)
-		p := newProcessor(newProcessorParams{
+		starting := make(chan *base.TaskMessage)
+		finished := make(chan *base.TaskMessage)
+		done := make(chan struct{})
+		defer func() { close(done) }()
+		go fakeHeartbeater(starting, finished, done)
+		p := newProcessor(processorParams{
 			logger:          testLogger,
 			broker:          nil,
-			ss:              ss,
 			retryDelayFunc:  defaultDelayFunc,
 			syncCh:          nil,
-			cancelations:    cancelations,
+			cancelations:    base.NewCancelations(),
+			concurrency:     10,
+			queues:          tc.queueCfg,
+			strictPriority:  false,
 			errHandler:      nil,
 			shutdownTimeout: defaultShutdownTimeout,
+			starting:        starting,
+			finished:        finished,
 		})
 		got := p.queues()
 		if diff := cmp.Diff(tc.want, got, sortOpt); diff != "" {
@@ -325,23 +354,28 @@ func TestProcessorWithStrictPriority(t *testing.T) {
 			base.DefaultQueueName: 2,
 			"low":                 1,
 		}
-		// Note: Set concurrency to 1 to make sure tasks are processed one at a time.
-		cancelations := base.NewCancelations()
-		ss := base.NewServerState("localhost", 1234, 1 /* concurrency */, queueCfg, true /*strict*/)
-		p := newProcessor(newProcessorParams{
+		starting := make(chan *base.TaskMessage)
+		finished := make(chan *base.TaskMessage)
+		done := make(chan struct{})
+		defer func() { close(done) }()
+		go fakeHeartbeater(starting, finished, done)
+		p := newProcessor(processorParams{
 			logger:          testLogger,
 			broker:          rdbClient,
-			ss:              ss,
 			retryDelayFunc:  defaultDelayFunc,
 			syncCh:          nil,
-			cancelations:    cancelations,
+			cancelations:    base.NewCancelations(),
+			concurrency:     1, // Set concurrency to 1 to make sure tasks are processed one at a time.
+			queues:          queueCfg,
+			strictPriority:  true,
 			errHandler:      nil,
 			shutdownTimeout: defaultShutdownTimeout,
+			starting:        starting,
+			finished:        finished,
 		})
 		p.handler = HandlerFunc(handler)
 
-		var wg sync.WaitGroup
-		p.start(&wg)
+		p.start(&sync.WaitGroup{})
 		time.Sleep(tc.wait)
 		p.terminate()
 
@@ -401,88 +435,6 @@ func TestPerform(t *testing.T) {
 	}
 }
 
-func TestCreateContextWithTimeRestrictions(t *testing.T) {
-	var (
-		noTimeout  = time.Duration(0)
-		noDeadline = time.Time{}
-	)
-
-	tests := []struct {
-		desc         string
-		timeout      time.Duration
-		deadline     time.Time
-		wantDeadline time.Time
-	}{
-		{"only with timeout", 10 * time.Second, noDeadline, time.Now().Add(10 * time.Second)},
-		{"only with deadline", noTimeout, time.Now().Add(time.Hour), time.Now().Add(time.Hour)},
-		{"with timeout and deadline (timeout < deadline)", 10 * time.Second, time.Now().Add(time.Hour), time.Now().Add(10 * time.Second)},
-		{"with timeout and deadline (timeout > deadline)", 10 * time.Minute, time.Now().Add(30 * time.Second), time.Now().Add(30 * time.Second)},
-	}
-
-	for _, tc := range tests {
-		msg := &base.TaskMessage{
-			Type:     "something",
-			ID:       xid.New(),
-			Timeout:  tc.timeout.String(),
-			Deadline: tc.deadline.Format(time.RFC3339),
-		}
-
-		ctx, cancel := createContext(msg)
-
-		select {
-		case x := <-ctx.Done():
-			t.Errorf("%s: <-ctx.Done() == %v, want nothing (it should block)", tc.desc, x)
-		default:
-		}
-
-		got, ok := ctx.Deadline()
-		if !ok {
-			t.Errorf("%s: ctx.Deadline() returned false, want deadline to be set", tc.desc)
-		}
-		if !cmp.Equal(tc.wantDeadline, got, cmpopts.EquateApproxTime(time.Second)) {
-			t.Errorf("%s: ctx.Deadline() returned %v, want %v", tc.desc, got, tc.wantDeadline)
-		}
-
-		cancel()
-
-		select {
-		case <-ctx.Done():
-		default:
-			t.Errorf("ctx.Done() blocked, want it to be non-blocking")
-		}
-	}
-}
-
-func TestCreateContextWithoutTimeRestrictions(t *testing.T) {
-	msg := &base.TaskMessage{
-		Type:     "something",
-		ID:       xid.New(),
-		Timeout:  time.Duration(0).String(),        // zero value to indicate no timeout
-		Deadline: time.Time{}.Format(time.RFC3339), // zero value to indicate no deadline
-	}
-
-	ctx, cancel := createContext(msg)
-
-	select {
-	case x := <-ctx.Done():
-		t.Errorf("<-ctx.Done() == %v, want nothing (it should block)", x)
-	default:
-	}
-
-	_, ok := ctx.Deadline()
-	if ok {
-		t.Error("ctx.Deadline() returned true, want deadline to not be set")
-	}
-
-	cancel()
-
-	select {
-	case <-ctx.Done():
-	default:
-		t.Error("ctx.Done() blocked, want it to be non-blocking")
-	}
-}
-
 func TestGCD(t *testing.T) {
 	tests := []struct {
 		input []int
@@ -505,7 +457,7 @@ func TestGCD(t *testing.T) {
 	}
 }
 
-func TestNormalizeQueueCfg(t *testing.T) {
+func TestNormalizeQueues(t *testing.T) {
 	tests := []struct {
 		input map[string]int
 		want  map[string]int
@@ -555,9 +507,9 @@ func TestNormalizeQueueCfg(t *testing.T) {
 	}
 
 	for _, tc := range tests {
-		got := normalizeQueueCfg(tc.input)
+		got := normalizeQueues(tc.input)
 		if diff := cmp.Diff(tc.want, got); diff != "" {
-			t.Errorf("normalizeQueueCfg(%v) = %v, want %v; (-want, +got):\n%s",
+			t.Errorf("normalizeQueues(%v) = %v, want %v; (-want, +got):\n%s",
 				tc.input, got, tc.want, diff)
 		}
 	}

scheduler.go

@@ -9,10 +9,11 @@ import (
 	"time"
 
 	"github.com/hibiken/asynq/internal/base"
+	"github.com/hibiken/asynq/internal/log"
 )
 
 type scheduler struct {
-	logger Logger
+	logger *log.Logger
 	broker base.Broker
 
 	// channel to communicate back to the long running "scheduler" goroutine.
@@ -20,27 +21,25 @@ type scheduler struct {
 
 	// poll interval on average
 	avgInterval time.Duration
-
-	// list of queues to move the tasks into.
-	qnames []string
 }
 
-func newScheduler(l Logger, b base.Broker, avgInterval time.Duration, qcfg map[string]int) *scheduler {
-	var qnames []string
-	for q := range qcfg {
-		qnames = append(qnames, q)
+type schedulerParams struct {
+	logger   *log.Logger
+	broker   base.Broker
+	interval time.Duration
 }
+
+func newScheduler(params schedulerParams) *scheduler {
 	return &scheduler{
-		logger:      l,
-		broker:      b,
+		logger:      params.logger,
+		broker:      params.broker,
 		done:        make(chan struct{}),
-		avgInterval: avgInterval,
-		qnames:      qnames,
+		avgInterval: params.interval,
 	}
 }
 
 func (s *scheduler) terminate() {
-	s.logger.Info("Scheduler shutting down...")
+	s.logger.Debug("Scheduler shutting down...")
 	// Signal the scheduler goroutine to stop polling.
 	s.done <- struct{}{}
 }
@@ -53,7 +52,7 @@ func (s *scheduler) start(wg *sync.WaitGroup) {
 		for {
 			select {
 			case <-s.done:
-				s.logger.Info("Scheduler done")
+				s.logger.Debug("Scheduler done")
 				return
 			case <-time.After(s.avgInterval):
 				s.exec()
@@ -63,7 +62,7 @@ func (s *scheduler) start(wg *sync.WaitGroup) {
 }
 
 func (s *scheduler) exec() {
-	if err := s.broker.CheckAndEnqueue(s.qnames...); err != nil {
-		s.logger.Error("Could not enqueue scheduled tasks: %v", err)
+	if err := s.broker.CheckAndEnqueue(); err != nil {
+		s.logger.Errorf("Could not enqueue scheduled tasks: %v", err)
 	}
 }

scheduler_test.go

@@ -19,7 +19,11 @@ func TestScheduler(t *testing.T) {
 	r := setup(t)
 	rdbClient := rdb.NewRDB(r)
 	const pollInterval = time.Second
-	s := newScheduler(testLogger, rdbClient, pollInterval, defaultQueueConfig)
+	s := newScheduler(schedulerParams{
+		logger:   testLogger,
+		broker:   rdbClient,
+		interval: pollInterval,
+	})
 	t1 := h.NewTaskMessage("gen_thumbnail", nil)
 	t2 := h.NewTaskMessage("send_email", nil)
 	t3 := h.NewTaskMessage("reindex", nil)

servemux.go

@@ -15,7 +15,7 @@ import (
 // ServeMux is a multiplexer for asynchronous tasks.
 // It matches the type of each task against a list of registered patterns
 // and calls the handler for the pattern that most closely matches the
-// taks's type name.
+// task's type name.
 //
 // Longer patterns take precedence over shorter ones, so that if there are
 // handlers registered for both "images" and "images:thumbnails",

server.go

@@ -10,8 +10,8 @@ import (
 	"fmt"
 	"math"
 	"math/rand"
-	"os"
 	"runtime"
+	"strings"
 	"sync"
 	"time"
 
@@ -33,12 +33,12 @@ import (
 // (e.g., queue size reaches a certain limit, or the task has been in the
 // queue for a certain amount of time).
 type Server struct {
-	ss *base.ServerState
-
-	logger Logger
+	logger *log.Logger
 
 	broker base.Broker
 
+	status *base.ServerStatus
+
 	// wait group to wait for all goroutines to finish.
 	wg          sync.WaitGroup
 	scheduler   *scheduler
@@ -112,6 +112,11 @@ type Config struct {
 	// If unset, default logger is used.
 	Logger Logger
 
+	// LogLevel specifies the minimum log level to enable.
+	//
+	// If unset, InfoLevel is used by default.
+	LogLevel LogLevel
+
 	// ShutdownTimeout specifies the duration to wait to let workers finish their tasks
 	// before forcing them to abort when stopping the server.
 	//
@@ -133,23 +138,104 @@ func (fn ErrorHandlerFunc) HandleError(task *Task, err error, retried, maxRetry
 	fn(task, err, retried, maxRetry)
 }
 
-// Logger implements logging with various log levels.
+// Logger supports logging at various log levels.
 type Logger interface {
 	// Debug logs a message at Debug level.
-	Debug(format string, args ...interface{})
+	Debug(args ...interface{})
 
 	// Info logs a message at Info level.
-	Info(format string, args ...interface{})
+	Info(args ...interface{})
 
 	// Warn logs a message at Warning level.
-	Warn(format string, args ...interface{})
+	Warn(args ...interface{})
 
 	// Error logs a message at Error level.
-	Error(format string, args ...interface{})
+	Error(args ...interface{})
 
 	// Fatal logs a message at Fatal level
 	// and process will exit with status set to 1.
-	Fatal(format string, args ...interface{})
+	Fatal(args ...interface{})
+}
+
+// LogLevel represents logging level.
+//
+// It satisfies flag.Value interface.
+type LogLevel int32
+
+const (
+	// Note: reserving value zero to differentiate unspecified case.
+	level_unspecified LogLevel = iota
+
+	// DebugLevel is the lowest level of logging.
+	// Debug logs are intended for debugging and development purposes.
+	DebugLevel
+
+	// InfoLevel is used for general informational log messages.
+	InfoLevel
+
+	// WarnLevel is used for undesired but relatively expected events,
+	// which may indicate a problem.
+	WarnLevel
+
+	// ErrorLevel is used for undesired and unexpected events that
+	// the program can recover from.
+	ErrorLevel
+
+	// FatalLevel is used for undesired and unexpected events that
+	// the program cannot recover from.
+	FatalLevel
+)
+
+// String is part of the flag.Value interface.
+func (l *LogLevel) String() string {
+	switch *l {
+	case DebugLevel:
+		return "debug"
+	case InfoLevel:
+		return "info"
+	case WarnLevel:
+		return "warn"
+	case ErrorLevel:
+		return "error"
+	case FatalLevel:
+		return "fatal"
+	}
+	panic(fmt.Sprintf("asynq: unexpected log level: %v", *l))
+}
+
+// Set is part of the flag.Value interface.
+func (l *LogLevel) Set(val string) error {
+	switch strings.ToLower(val) {
+	case "debug":
+		*l = DebugLevel
+	case "info":
+		*l = InfoLevel
+	case "warn", "warning":
+		*l = WarnLevel
+	case "error":
+		*l = ErrorLevel
+	case "fatal":
+		*l = FatalLevel
+	default:
+		return fmt.Errorf("asynq: unsupported log level %q", val)
+	}
+	return nil
+}
+
+func toInternalLogLevel(l LogLevel) log.Level {
+	switch l {
+	case DebugLevel:
+		return log.DebugLevel
+	case InfoLevel:
+		return log.InfoLevel
+	case WarnLevel:
+		return log.WarnLevel
+	case ErrorLevel:
+		return log.ErrorLevel
+	case FatalLevel:
+		return log.FatalLevel
+	}
+	panic(fmt.Sprintf("asynq: unexpected log level: %v", l))
 }
 
 // Formula taken from https://github.com/mperham/sidekiq.
@@ -185,43 +271,68 @@ func NewServer(r RedisConnOpt, cfg Config) *Server {
 	if len(queues) == 0 {
 		queues = defaultQueueConfig
 	}
-	logger := cfg.Logger
-	if logger == nil {
-		logger = log.NewLogger(os.Stderr)
-	}
 	shutdownTimeout := cfg.ShutdownTimeout
 	if shutdownTimeout == 0 {
 		shutdownTimeout = defaultShutdownTimeout
 	}
-
-	host, err := os.Hostname()
-	if err != nil {
-		host = "unknown-host"
+	logger := log.NewLogger(cfg.Logger)
+	loglevel := cfg.LogLevel
+	if loglevel == level_unspecified {
+		loglevel = InfoLevel
 	}
-	pid := os.Getpid()
+	logger.SetLevel(toInternalLogLevel(loglevel))
 
 	rdb := rdb.NewRDB(createRedisClient(r))
-	ss := base.NewServerState(host, pid, n, queues, cfg.StrictPriority)
+	starting := make(chan *base.TaskMessage)
+	finished := make(chan *base.TaskMessage)
 	syncCh := make(chan *syncRequest)
+	status := base.NewServerStatus(base.StatusIdle)
 	cancels := base.NewCancelations()
-	syncer := newSyncer(logger, syncCh, 5*time.Second)
-	heartbeater := newHeartbeater(logger, rdb, ss, 5*time.Second)
-	scheduler := newScheduler(logger, rdb, 5*time.Second, queues)
-	subscriber := newSubscriber(logger, rdb, cancels)
-	processor := newProcessor(newProcessorParams{
+
+	syncer := newSyncer(syncerParams{
+		logger:     logger,
+		requestsCh: syncCh,
+		interval:   5 * time.Second,
+	})
+	heartbeater := newHeartbeater(heartbeaterParams{
+		logger:         logger,
+		broker:         rdb,
+		interval:       5 * time.Second,
+		concurrency:    n,
+		queues:         queues,
+		strictPriority: cfg.StrictPriority,
+		status:         status,
+		starting:       starting,
+		finished:       finished,
+	})
+	scheduler := newScheduler(schedulerParams{
+		logger:   logger,
+		broker:   rdb,
+		interval: 5 * time.Second,
+	})
+	subscriber := newSubscriber(subscriberParams{
+		logger:       logger,
+		broker:       rdb,
+		cancelations: cancels,
+	})
+	processor := newProcessor(processorParams{
 		logger:          logger,
 		broker:          rdb,
-		ss:              ss,
 		retryDelayFunc:  delayFunc,
 		syncCh:          syncCh,
 		cancelations:    cancels,
+		concurrency:     n,
+		queues:          queues,
+		strictPriority:  cfg.StrictPriority,
 		errHandler:      cfg.ErrorHandler,
 		shutdownTimeout: shutdownTimeout,
+		starting:        starting,
+		finished:        finished,
 	})
 	return &Server{
-		ss:          ss,
 		logger:      logger,
 		broker:      rdb,
+		status:      status,
 		scheduler:   scheduler,
 		processor:   processor,
 		syncer:      syncer,
@@ -282,22 +393,15 @@ func (srv *Server) Start(handler Handler) error {
 	if handler == nil {
 		return fmt.Errorf("asynq: server cannot run with nil handler")
 	}
-	switch srv.ss.Status() {
+	switch srv.status.Get() {
 	case base.StatusRunning:
 		return fmt.Errorf("asynq: the server is already running")
 	case base.StatusStopped:
 		return ErrServerStopped
 	}
-	srv.ss.SetStatus(base.StatusRunning)
+	srv.status.Set(base.StatusRunning)
 	srv.processor.handler = handler
 
-	type prefixLogger interface {
-		SetPrefix(prefix string)
-	}
-	// If logger supports setting prefix, then set prefix for log output.
-	if l, ok := srv.logger.(prefixLogger); ok {
-		l.SetPrefix(fmt.Sprintf("asynq: pid=%d ", os.Getpid()))
-	}
 	srv.logger.Info("Starting processing")
 
 	srv.heartbeater.start(&srv.wg)
@@ -313,17 +417,17 @@ func (srv *Server) Start(handler Handler) error {
 // active workers to finish processing tasks for duration specified in Config.ShutdownTimeout.
 // If worker didn't finish processing a task during the timeout, the task will be pushed back to Redis.
 func (srv *Server) Stop() {
-	switch srv.ss.Status() {
+	switch srv.status.Get() {
 	case base.StatusIdle, base.StatusStopped:
 		// server is not running, do nothing and return.
 		return
 	}
 
-	fmt.Println() // print newline for prettier log.
 	srv.logger.Info("Starting graceful shutdown")
 	// Note: The order of termination is important.
 	// Sender goroutines should be terminated before the receiver goroutines.
 	// processor -> syncer (via syncCh)
+	// processor -> heartbeater (via starting, finished channels)
 	srv.scheduler.terminate()
 	srv.processor.terminate()
 	srv.syncer.terminate()
@@ -333,14 +437,16 @@ func (srv *Server) Stop() {
 	srv.wg.Wait()
 
 	srv.broker.Close()
-	srv.ss.SetStatus(base.StatusStopped)
+	srv.status.Set(base.StatusStopped)
 
-	srv.logger.Info("Bye!")
+	srv.logger.Info("Exiting")
 }
 
 // Quiet signals the server to stop pulling new tasks off queues.
 // Quiet should be used before stopping the server.
 func (srv *Server) Quiet() {
+	srv.logger.Info("Stopping processor")
 	srv.processor.stop()
-	srv.ss.SetStatus(base.StatusQuiet)
+	srv.status.Set(base.StatusQuiet)
+	srv.logger.Info("Processor stopped")
 }

server_test.go

@@ -6,9 +6,13 @@ package asynq
 
 import (
 	"context"
+	"fmt"
+	"syscall"
 	"testing"
 	"time"
 
+	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/hibiken/asynq/internal/testbroker"
 	"go.uber.org/goleak"
 )
 
@@ -24,6 +28,7 @@ func TestServer(t *testing.T) {
 	c := NewClient(r)
 	srv := NewServer(r, Config{
 		Concurrency: 10,
+		LogLevel:    testLogLevel,
 	})
 
 	// no-op handler
@@ -49,8 +54,37 @@ func TestServer(t *testing.T) {
 	srv.Stop()
 }
 
+func TestServerRun(t *testing.T) {
+	// https://github.com/go-redis/redis/issues/1029
+	ignoreOpt := goleak.IgnoreTopFunction("github.com/go-redis/redis/v7/internal/pool.(*ConnPool).reaper")
+	defer goleak.VerifyNoLeaks(t, ignoreOpt)
+
+	srv := NewServer(RedisClientOpt{Addr: ":6379"}, Config{LogLevel: testLogLevel})
+
+	done := make(chan struct{})
+	// Make sure server exits when receiving TERM signal.
+	go func() {
+		time.Sleep(2 * time.Second)
+		syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		done <- struct{}{}
+	}()
+
+	go func() {
+		select {
+		case <-time.After(10 * time.Second):
+			t.Fatal("server did not stop after receiving TERM signal")
+		case <-done:
+		}
+	}()
+
+	mux := NewServeMux()
+	if err := srv.Run(mux); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestServerErrServerStopped(t *testing.T) {
-	srv := NewServer(RedisClientOpt{Addr: ":6379"}, Config{})
+	srv := NewServer(RedisClientOpt{Addr: ":6379"}, Config{LogLevel: testLogLevel})
 	handler := NewServeMux()
 	if err := srv.Start(handler); err != nil {
 		t.Fatal(err)
@@ -63,7 +97,7 @@ func TestServerErrServerStopped(t *testing.T) {
 }
 
 func TestServerErrNilHandler(t *testing.T) {
-	srv := NewServer(RedisClientOpt{Addr: ":6379"}, Config{})
+	srv := NewServer(RedisClientOpt{Addr: ":6379"}, Config{LogLevel: testLogLevel})
 	err := srv.Start(nil)
 	if err == nil {
 		t.Error("Starting server with nil handler: (*Server).Start(nil) did not return error")
@@ -72,7 +106,7 @@ func TestServerErrNilHandler(t *testing.T) {
 }
 
 func TestServerErrServerRunning(t *testing.T) {
-	srv := NewServer(RedisClientOpt{Addr: ":6379"}, Config{})
+	srv := NewServer(RedisClientOpt{Addr: ":6379"}, Config{LogLevel: testLogLevel})
 	handler := NewServeMux()
 	if err := srv.Start(handler); err != nil {
 		t.Fatal(err)
@@ -83,3 +117,124 @@ func TestServerErrServerRunning(t *testing.T) {
 	}
 	srv.Stop()
 }
+
+func TestServerWithRedisDown(t *testing.T) {
+	// Make sure that server does not panic and exit if redis is down.
+	defer func() {
+		if r := recover(); r != nil {
+			t.Errorf("panic occurred: %v", r)
+		}
+	}()
+	r := rdb.NewRDB(setup(t))
+	testBroker := testbroker.NewTestBroker(r)
+	srv := NewServer(RedisClientOpt{Addr: ":6379"}, Config{LogLevel: testLogLevel})
+	srv.broker = testBroker
+	srv.scheduler.broker = testBroker
+	srv.heartbeater.broker = testBroker
+	srv.processor.broker = testBroker
+	srv.subscriber.broker = testBroker
+	testBroker.Sleep()
+
+	// no-op handler
+	h := func(ctx context.Context, task *Task) error {
+		return nil
+	}
+
+	err := srv.Start(HandlerFunc(h))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	time.Sleep(3 * time.Second)
+
+	srv.Stop()
+}
+
+func TestServerWithFlakyBroker(t *testing.T) {
+	// Make sure that server does not panic and exit if redis is down.
+	defer func() {
+		if r := recover(); r != nil {
+			t.Errorf("panic occurred: %v", r)
+		}
+	}()
+	r := rdb.NewRDB(setup(t))
+	testBroker := testbroker.NewTestBroker(r)
+	srv := NewServer(RedisClientOpt{Addr: redisAddr, DB: redisDB}, Config{LogLevel: testLogLevel})
+	srv.broker = testBroker
+	srv.scheduler.broker = testBroker
+	srv.heartbeater.broker = testBroker
+	srv.processor.broker = testBroker
+	srv.subscriber.broker = testBroker
+
+	c := NewClient(RedisClientOpt{Addr: redisAddr, DB: redisDB})
+
+	h := func(ctx context.Context, task *Task) error {
+		// force task retry.
+		if task.Type == "bad_task" {
+			return fmt.Errorf("could not process %q", task.Type)
+		}
+		time.Sleep(2 * time.Second)
+		return nil
+	}
+
+	err := srv.Start(HandlerFunc(h))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < 10; i++ {
+		err := c.Enqueue(NewTask("enqueued", nil), MaxRetry(i))
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = c.Enqueue(NewTask("bad_task", nil))
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = c.EnqueueIn(time.Duration(i)*time.Second, NewTask("scheduled", nil))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// simulate redis going down.
+	testBroker.Sleep()
+
+	time.Sleep(3 * time.Second)
+
+	// simulate redis comes back online.
+	testBroker.Wakeup()
+
+	time.Sleep(3 * time.Second)
+
+	srv.Stop()
+}
+
+func TestLogLevel(t *testing.T) {
+	tests := []struct {
+		flagVal string
+		want    LogLevel
+		wantStr string
+	}{
+		{"debug", DebugLevel, "debug"},
+		{"Info", InfoLevel, "info"},
+		{"WARN", WarnLevel, "warn"},
+		{"warning", WarnLevel, "warn"},
+		{"Error", ErrorLevel, "error"},
+		{"fatal", FatalLevel, "fatal"},
+	}
+
+	for _, tc := range tests {
+		level := new(LogLevel)
+		if err := level.Set(tc.flagVal); err != nil {
+			t.Fatal(err)
+		}
+		if *level != tc.want {
+			t.Errorf("Set(%q): got %v, want %v", tc.flagVal, level, &tc.want)
+			continue
+		}
+		if got := level.String(); got != tc.wantStr {
+			t.Errorf("String() returned %q, want %q", got, tc.wantStr)
+		}
+	}
+}

subscriber.go

@@ -10,10 +10,11 @@ import (
 
 	"github.com/go-redis/redis/v7"
 	"github.com/hibiken/asynq/internal/base"
+	"github.com/hibiken/asynq/internal/log"
 )
 
 type subscriber struct {
-	logger Logger
+	logger *log.Logger
 	broker base.Broker
 
 	// channel to communicate back to the long running "subscriber" goroutine.
@@ -26,18 +27,24 @@ type subscriber struct {
 	retryTimeout time.Duration
 }
 
-func newSubscriber(l Logger, b base.Broker, cancelations *base.Cancelations) *subscriber {
+type subscriberParams struct {
+	logger       *log.Logger
+	broker       base.Broker
+	cancelations *base.Cancelations
+}
+
+func newSubscriber(params subscriberParams) *subscriber {
 	return &subscriber{
-		logger:       l,
-		broker:       b,
+		logger:       params.logger,
+		broker:       params.broker,
 		done:         make(chan struct{}),
-		cancelations: cancelations,
+		cancelations: params.cancelations,
 		retryTimeout: 5 * time.Second,
 	}
 }
 
 func (s *subscriber) terminate() {
-	s.logger.Info("Subscriber shutting down...")
+	s.logger.Debug("Subscriber shutting down...")
 	// Signal the subscriber goroutine to stop.
 	s.done <- struct{}{}
 }
@@ -54,12 +61,12 @@ func (s *subscriber) start(wg *sync.WaitGroup) {
 		for {
 			pubsub, err = s.broker.CancelationPubSub()
 			if err != nil {
-				s.logger.Error("cannot subscribe to cancelation channel: %v", err)
+				s.logger.Errorf("cannot subscribe to cancelation channel: %v", err)
 				select {
 				case <-time.After(s.retryTimeout):
 					continue
 				case <-s.done:
-					s.logger.Info("Subscriber done")
+					s.logger.Debug("Subscriber done")
 					return
 				}
 			}
@@ -70,7 +77,7 @@ func (s *subscriber) start(wg *sync.WaitGroup) {
 			select {
 			case <-s.done:
 				pubsub.Close()
-				s.logger.Info("Subscriber done")
+				s.logger.Debug("Subscriber done")
 				return
 			case msg := <-cancelCh:
 				cancel, ok := s.cancelations.Get(msg.Payload)

subscriber_test.go

@@ -38,7 +38,11 @@ func TestSubscriber(t *testing.T) {
 		cancelations := base.NewCancelations()
 		cancelations.Add(tc.registeredID, fakeCancelFunc)
 
-		subscriber := newSubscriber(testLogger, rdbClient, cancelations)
+		subscriber := newSubscriber(subscriberParams{
+			logger:       testLogger,
+			broker:       rdbClient,
+			cancelations: cancelations,
+		})
 		var wg sync.WaitGroup
 		subscriber.start(&wg)
 		defer subscriber.terminate()
@@ -75,7 +79,11 @@ func TestSubscriberWithRedisDown(t *testing.T) {
 	testBroker := testbroker.NewTestBroker(r)
 
 	cancelations := base.NewCancelations()
-	subscriber := newSubscriber(testLogger, testBroker, cancelations)
+	subscriber := newSubscriber(subscriberParams{
+		logger:       testLogger,
+		broker:       testBroker,
+		cancelations: cancelations,
+	})
 	subscriber.retryTimeout = 1 * time.Second // set shorter retry timeout for testing purpose.
 
 	testBroker.Sleep() // simulate a situation where subscriber cannot connect to redis.

syncer.go

@@ -7,12 +7,14 @@ package asynq
 import (
 	"sync"
 	"time"
+
+	"github.com/hibiken/asynq/internal/log"
 )
 
 // syncer is responsible for queuing up failed requests to redis and retry
 // those requests to sync state between the background process and redis.
 type syncer struct {
-	logger Logger
+	logger *log.Logger
 
 	requestsCh <-chan *syncRequest
 
@@ -28,17 +30,23 @@ type syncRequest struct {
 	errMsg string       // error message
 }
 
-func newSyncer(l Logger, requestsCh <-chan *syncRequest, interval time.Duration) *syncer {
+type syncerParams struct {
+	logger     *log.Logger
+	requestsCh <-chan *syncRequest
+	interval   time.Duration
+}
+
+func newSyncer(params syncerParams) *syncer {
 	return &syncer{
-		logger:     l,
-		requestsCh: requestsCh,
+		logger:     params.logger,
+		requestsCh: params.requestsCh,
 		done:       make(chan struct{}),
-		interval:   interval,
+		interval:   params.interval,
 	}
 }
 
 func (s *syncer) terminate() {
-	s.logger.Info("Syncer shutting down...")
+	s.logger.Debug("Syncer shutting down...")
 	// Signal the syncer goroutine to stop.
 	s.done <- struct{}{}
 }
@@ -57,7 +65,7 @@ func (s *syncer) start(wg *sync.WaitGroup) {
 						s.logger.Error(req.errMsg)
 					}
 				}
-				s.logger.Info("Syncer done")
+				s.logger.Debug("Syncer done")
 				return
 			case req := <-s.requestsCh:
 				requests = append(requests, req)

syncer_test.go

@@ -27,7 +27,11 @@ func TestSyncer(t *testing.T) {
 
 	const interval = time.Second
 	syncRequestCh := make(chan *syncRequest)
-	syncer := newSyncer(testLogger, syncRequestCh, interval)
+	syncer := newSyncer(syncerParams{
+		logger:     testLogger,
+		requestsCh: syncRequestCh,
+		interval:   interval,
+	})
 	var wg sync.WaitGroup
 	syncer.start(&wg)
 	defer syncer.terminate()
@@ -52,7 +56,11 @@ func TestSyncer(t *testing.T) {
 func TestSyncerRetry(t *testing.T) {
 	const interval = time.Second
 	syncRequestCh := make(chan *syncRequest)
-	syncer := newSyncer(testLogger, syncRequestCh, interval)
+	syncer := newSyncer(syncerParams{
+		logger:     testLogger,
+		requestsCh: syncRequestCh,
+		interval:   interval,
+	})
 
 	var wg sync.WaitGroup
 	syncer.start(&wg)

tools/asynq/README.md

@@ -14,6 +14,7 @@ Asynq CLI is a command line tool to monitor the tasks managed by `asynq` package
   - [Delete](#delete)
   - [Kill](#kill)
   - [Cancel](#cancel)
+  - [Pause](#pause)
 - [Config File](#config-file)
 
 ## Installation
@@ -144,6 +145,17 @@ Example:
 
     asynq cancel bnogo8gt6toe23vhef0g
 
+### Pause
+
+Command `pause` pauses the spcified queue. Tasks in paused queues are not processed by servers.
+To resume processing from the queue, use `unpause` command.
+To see which queues are currently paused, use `stats` command.
+
+Example:
+
+    asynq pause email
+    asynq unpause email
+
 ## Config File
 
 You can use a config file to set default values for the flags.

tools/asynq/cmd/pause.go

@@ -0,0 +1,47 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/go-redis/redis/v7"
+	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// pauseCmd represents the pause command
+var pauseCmd = &cobra.Command{
+	Use:   "pause [queue name]",
+	Short: "Pauses the specified queue",
+	Long: `Pause (asynq pause) will pause the specified queue.
+Asynq servers will not process tasks from paused queues.
+Use the "unpause" command to resume a paused queue.
+
+Example: asynq pause default -> Pause the "default" queue`,
+	Args: cobra.ExactValidArgs(1),
+	Run:  pause,
+}
+
+func init() {
+	rootCmd.AddCommand(pauseCmd)
+}
+
+func pause(cmd *cobra.Command, args []string) {
+	c := redis.NewClient(&redis.Options{
+		Addr:     viper.GetString("uri"),
+		DB:       viper.GetInt("db"),
+		Password: viper.GetString("password"),
+	})
+	r := rdb.NewRDB(c)
+	err := r.Pause(args[0])
+	if err != nil {
+		fmt.Printf("error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("Successfully paused queue %q\n", args[0])
+}

tools/asynq/cmd/stats.go

@@ -7,7 +7,6 @@ package cmd
 import (
 	"fmt"
 	"os"
-	"sort"
 	"strconv"
 	"strings"
 	"text/tabwriter"
@@ -96,24 +95,31 @@ func printStates(s *rdb.Stats) {
 	tw.Flush()
 }
 
-func printQueues(queues map[string]int) {
-	var qnames, seps, counts []string
-	for q := range queues {
-		qnames = append(qnames, strings.Title(q))
+func printQueues(queues []*rdb.Queue) {
+	var headers, seps, counts []string
+	for _, q := range queues {
+		title := queueTitle(q)
+		headers = append(headers, title)
+		seps = append(seps, strings.Repeat("-", len(title)))
+		counts = append(counts, strconv.Itoa(q.Size))
 	}
-	sort.Strings(qnames) // sort for stable order
-	for _, q := range qnames {
-		seps = append(seps, strings.Repeat("-", len(q)))
-		counts = append(counts, strconv.Itoa(queues[strings.ToLower(q)]))
-	}
-	format := strings.Repeat("%v\t", len(qnames)) + "\n"
+	format := strings.Repeat("%v\t", len(headers)) + "\n"
 	tw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0)
-	fmt.Fprintf(tw, format, toInterfaceSlice(qnames)...)
+	fmt.Fprintf(tw, format, toInterfaceSlice(headers)...)
 	fmt.Fprintf(tw, format, toInterfaceSlice(seps)...)
 	fmt.Fprintf(tw, format, toInterfaceSlice(counts)...)
 	tw.Flush()
 }
 
+func queueTitle(q *rdb.Queue) string {
+	var b strings.Builder
+	b.WriteString(strings.Title(q.Name))
+	if q.Paused {
+		b.WriteString(" (Paused)")
+	}
+	return b.String()
+}
+
 func printStats(s *rdb.Stats) {
 	format := strings.Repeat("%v\t", 3) + "\n"
 	tw := new(tabwriter.Writer).Init(os.Stdout, 0, 8, 2, ' ', 0)

tools/asynq/cmd/unpause.go

@@ -0,0 +1,46 @@
+// Copyright 2020 Kentaro Hibino. All rights reserved.
+// Use of this source code is governed by a MIT license
+// that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/go-redis/redis/v7"
+	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// unpauseCmd represents the unpause command
+var unpauseCmd = &cobra.Command{
+	Use:   "unpause [queue name]",
+	Short: "Unpauses the specified queue",
+	Long: `Unpause (asynq unpause) will unpause the specified queue.
+Asynq servers will process tasks from unpaused/resumed queues.
+
+Example: asynq unpause default -> Resume the "default" queue`,
+	Args: cobra.ExactValidArgs(1),
+	Run:  unpause,
+}
+
+func init() {
+	rootCmd.AddCommand(unpauseCmd)
+}
+
+func unpause(cmd *cobra.Command, args []string) {
+	c := redis.NewClient(&redis.Options{
+		Addr:     viper.GetString("uri"),
+		DB:       viper.GetInt("db"),
+		Password: viper.GetString("password"),
+	})
+	r := rdb.NewRDB(c)
+	err := r.Unpause(args[0])
+	if err != nil {
+		fmt.Printf("error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("Successfully resumed queue %q\n", args[0])
+}

tools/asynq/cmd/workers.go

@@ -61,7 +61,7 @@ func workers(cmd *cobra.Command, args []string) {
 		if x.Started != y.Started {
 			return x.Started.Before(y.Started)
 		}
-		return x.ID.String() < y.ID.String()
+		return x.ID < y.ID
 	})
 
 	cols := []string{"Process", "ID", "Type", "Payload", "Queue", "Started"}