Add test to simulate situation where redis is down

subscriber_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/hibiken/asynq/internal/base"
 	"github.com/hibiken/asynq/internal/rdb"
+	"github.com/hibiken/asynq/internal/testbroker"
 )
 
 func TestSubscriber(t *testing.T) {
@@ -40,12 +41,12 @@ func TestSubscriber(t *testing.T) {
 		subscriber := newSubscriber(testLogger, rdbClient, cancelations)
 		var wg sync.WaitGroup
 		subscriber.start(&wg)
+		defer subscriber.terminate()
 
 		// wait for subscriber to establish connection to pubsub channel
 		time.Sleep(time.Second)
 
 		if err := rdbClient.PublishCancelation(tc.publishID); err != nil {
-			subscriber.terminate()
 			t.Fatalf("could not publish cancelation message: %v", err)
 		}
 
@@ -61,7 +62,53 @@ func TestSubscriber(t *testing.T) {
 			}
 		}
 		mu.Unlock()
-
-		subscriber.terminate()
 	}
 }
+
+func TestSubscriberWithRedisDown(t *testing.T) {
+	defer func() {
+		if r := recover(); r != nil {
+			t.Errorf("panic occurred: %v", r)
+		}
+	}()
+	r := rdb.NewRDB(setup(t))
+	testBroker := testbroker.NewTestBroker(r)
+
+	cancelations := base.NewCancelations()
+	subscriber := newSubscriber(testLogger, testBroker, cancelations)
+	subscriber.retryTimeout = 1 * time.Second // set shorter retry timeout for testing purpose.
+
+	testBroker.Sleep() // simulate a situation where subscriber cannot connect to redis.
+	var wg sync.WaitGroup
+	subscriber.start(&wg)
+	defer subscriber.terminate()
+
+	time.Sleep(2 * time.Second) // subscriber should wait and retry connecting to redis.
+
+	testBroker.Wakeup() // simulate a situation where redis server is back online.
+
+	time.Sleep(2 * time.Second) // allow subscriber to establish pubsub channel.
+
+	const id = "test"
+	var (
+		mu     sync.Mutex
+		called bool
+	)
+	cancelations.Add(id, func() {
+		mu.Lock()
+		defer mu.Unlock()
+		called = true
+	})
+
+	if err := r.PublishCancelation(id); err != nil {
+		t.Fatalf("could not publish cancelation message: %v", err)
+	}
+
+	time.Sleep(time.Second) // wait for redis to publish message.
+
+	mu.Lock()
+	if !called {
+		t.Errorf("cancel function was not called")
+	}
+	mu.Unlock()
+}