Introduce Task Results

* Added Retention Option to specify retention TTL for tasks * Added ResultWriter as a client interface to write result data for the associated task
2025-09-19 05:17:30 +08:00 · 2021-11-05 16:52:54 -07:00
parent 4638405cbd
commit f4ddac4dcc
33 changed files with 2099 additions and 846 deletions
--- a/inspector_test.go
+++ b/inspector_test.go
@@ -276,6 +276,7 @@ func TestInspectorGetQueueInfo(t *testing.T) {
 		scheduled map[string][]base.Z
 		retry     map[string][]base.Z
 		archived  map[string][]base.Z
+		completed map[string][]base.Z
 		processed map[string]int
 		failed    map[string]int
 		qname     string
@@ -310,6 +311,11 @@ func TestInspectorGetQueueInfo(t *testing.T) {
 				"critical": {},
 				"low":      {},
 			},
+			completed: map[string][]base.Z{
+				"default":  {},
+				"critical": {},
+				"low":      {},
+			},
 			processed: map[string]int{
 				"default":  120,
 				"critical": 100,
@@ -329,6 +335,7 @@ func TestInspectorGetQueueInfo(t *testing.T) {
 				Scheduled: 2,
 				Retry:     0,
 				Archived:  0,
+				Completed: 0,
 				Processed: 120,
 				Failed:    2,
 				Paused:    false,
@@ -344,6 +351,7 @@ func TestInspectorGetQueueInfo(t *testing.T) {
 		h.SeedAllScheduledQueues(t, r, tc.scheduled)
 		h.SeedAllRetryQueues(t, r, tc.retry)
 		h.SeedAllArchivedQueues(t, r, tc.archived)
+		h.SeedAllCompletedQueues(t, r, tc.completed)
 		for qname, n := range tc.processed {
 			processedKey := base.ProcessedKey(qname, now)
 			r.Set(context.Background(), processedKey, n, 0)
@@ -424,7 +432,7 @@ func TestInspectorHistory(t *testing.T) {
 }

 func createPendingTask(msg *base.TaskMessage) *TaskInfo {
-	return newTaskInfo(msg, base.TaskStatePending, time.Now())
+	return newTaskInfo(msg, base.TaskStatePending, time.Now(), nil)
 }

 func TestInspectorGetTaskInfo(t *testing.T) {
@@ -489,6 +497,7 @@ func TestInspectorGetTaskInfo(t *testing.T) {
 				m1,
 				base.TaskStateActive,
 				time.Time{}, // zero value for n/a
+				nil,
 			),
 		},
 		{
@@ -498,6 +507,7 @@ func TestInspectorGetTaskInfo(t *testing.T) {
 				m2,
 				base.TaskStateScheduled,
 				fiveMinsFromNow,
+				nil,
 			),
 		},
 		{
@@ -507,6 +517,7 @@ func TestInspectorGetTaskInfo(t *testing.T) {
 				m3,
 				base.TaskStateRetry,
 				oneHourFromNow,
+				nil,
 			),
 		},
 		{
@@ -516,6 +527,7 @@ func TestInspectorGetTaskInfo(t *testing.T) {
 				m4,
 				base.TaskStateArchived,
 				time.Time{}, // zero value for n/a
+				nil,
 			),
 		},
 		{
@@ -525,6 +537,7 @@ func TestInspectorGetTaskInfo(t *testing.T) {
 				m5,
 				base.TaskStatePending,
 				now,
+				nil,
 			),
 		},
 	}
@@ -722,8 +735,8 @@ func TestInspectorListActiveTasks(t *testing.T) {
 			},
 			qname: "default",
 			want: []*TaskInfo{
-				newTaskInfo(m1, base.TaskStateActive, time.Time{}),
-				newTaskInfo(m2, base.TaskStateActive, time.Time{}),
+				newTaskInfo(m1, base.TaskStateActive, time.Time{}, nil),
+				newTaskInfo(m2, base.TaskStateActive, time.Time{}, nil),
 			},
 		},
 	}
@@ -749,6 +762,7 @@ func createScheduledTask(z base.Z) *TaskInfo {
 		z.Message,
 		base.TaskStateScheduled,
 		time.Unix(z.Score, 0),
+		nil,
 	)
 }

@@ -818,6 +832,7 @@ func createRetryTask(z base.Z) *TaskInfo {
 		z.Message,
 		base.TaskStateRetry,
 		time.Unix(z.Score, 0),
+		nil,
 	)
 }

@@ -888,6 +903,7 @@ func createArchivedTask(z base.Z) *TaskInfo {
 		z.Message,
 		base.TaskStateArchived,
 		time.Time{}, // zero value for n/a
+		nil,
 	)
 }

@@ -952,6 +968,83 @@ func TestInspectorListArchivedTasks(t *testing.T) {
 	}
 }

+func newCompletedTaskMessage(typename, qname string, retention time.Duration, completedAt time.Time) *base.TaskMessage {
+	msg := h.NewTaskMessageWithQueue(typename, nil, qname)
+	msg.Retention = int64(retention.Seconds())
+	msg.CompletedAt = completedAt.Unix()
+	return msg
+}
+
+func createCompletedTask(z base.Z) *TaskInfo {
+	return newTaskInfo(
+		z.Message,
+		base.TaskStateCompleted,
+		time.Time{}, // zero value for n/a
+		nil,         // TODO: Test with result data
+	)
+}
+
+func TestInspectorListCompletedTasks(t *testing.T) {
+	r := setup(t)
+	defer r.Close()
+	now := time.Now()
+	m1 := newCompletedTaskMessage("task1", "default", 1*time.Hour, now.Add(-3*time.Minute))     // Expires in 57 mins
+	m2 := newCompletedTaskMessage("task2", "default", 30*time.Minute, now.Add(-10*time.Minute)) // Expires in 20 mins
+	m3 := newCompletedTaskMessage("task3", "default", 2*time.Hour, now.Add(-30*time.Minute))    // Expires in 90 mins
+	m4 := newCompletedTaskMessage("task4", "custom", 15*time.Minute, now.Add(-2*time.Minute))   // Expires in 13 mins
+	z1 := base.Z{Message: m1, Score: m1.CompletedAt + m1.Retention}
+	z2 := base.Z{Message: m2, Score: m2.CompletedAt + m2.Retention}
+	z3 := base.Z{Message: m3, Score: m3.CompletedAt + m3.Retention}
+	z4 := base.Z{Message: m4, Score: m4.CompletedAt + m4.Retention}
+
+	inspector := NewInspector(getRedisConnOpt(t))
+
+	tests := []struct {
+		desc      string
+		completed map[string][]base.Z
+		qname     string
+		want      []*TaskInfo
+	}{
+		{
+			desc: "with a few completed tasks",
+			completed: map[string][]base.Z{
+				"default": {z1, z2, z3},
+				"custom":  {z4},
+			},
+			qname: "default",
+			// Should be sorted by expiration time (CompletedAt + Retention).
+			want: []*TaskInfo{
+				createCompletedTask(z2),
+				createCompletedTask(z1),
+				createCompletedTask(z3),
+			},
+		},
+		{
+			desc: "with empty completed queue",
+			completed: map[string][]base.Z{
+				"default": {},
+			},
+			qname: "default",
+			want:  []*TaskInfo(nil),
+		},
+	}
+
+	for _, tc := range tests {
+		h.FlushDB(t, r)
+		h.SeedAllCompletedQueues(t, r, tc.completed)
+
+		got, err := inspector.ListCompletedTasks(tc.qname)
+		if err != nil {
+			t.Errorf("%s; ListCompletedTasks(%q) returned error: %v", tc.desc, tc.qname, err)
+			continue
+		}
+		if diff := cmp.Diff(tc.want, got, cmp.AllowUnexported(TaskInfo{})); diff != "" {
+			t.Errorf("%s; ListCompletedTasks(%q) = %v, want %v; (-want,+got)\n%s",
+				tc.desc, tc.qname, got, tc.want, diff)
+		}
+	}
+}
+
 func TestInspectorListPagination(t *testing.T) {
 	// Create 100 tasks.
 	var msgs []*base.TaskMessage
@@ -1050,6 +1143,9 @@ func TestInspectorListTasksQueueNotFoundError(t *testing.T) {
 		if _, err := inspector.ListArchivedTasks(tc.qname); !errors.Is(err, tc.wantErr) {
 			t.Errorf("ListArchivedTasks(%q) returned error %v, want %v", tc.qname, err, tc.wantErr)
 		}
+		if _, err := inspector.ListCompletedTasks(tc.qname); !errors.Is(err, tc.wantErr) {
+			t.Errorf("ListCompletedTasks(%q) returned error %v, want %v", tc.qname, err, tc.wantErr)
+		}
 	}
 }

@@ -1315,6 +1411,72 @@ func TestInspectorDeleteAllArchivedTasks(t *testing.T) {
 	}
 }

+func TestInspectorDeleteAllCompletedTasks(t *testing.T) {
+	r := setup(t)
+	defer r.Close()
+	now := time.Now()
+	m1 := newCompletedTaskMessage("task1", "default", 30*time.Minute, now.Add(-2*time.Minute))
+	m2 := newCompletedTaskMessage("task2", "default", 30*time.Minute, now.Add(-5*time.Minute))
+	m3 := newCompletedTaskMessage("task3", "default", 30*time.Minute, now.Add(-10*time.Minute))
+	m4 := newCompletedTaskMessage("task4", "custom", 30*time.Minute, now.Add(-3*time.Minute))
+	z1 := base.Z{Message: m1, Score: m1.CompletedAt + m1.Retention}
+	z2 := base.Z{Message: m2, Score: m2.CompletedAt + m2.Retention}
+	z3 := base.Z{Message: m3, Score: m3.CompletedAt + m3.Retention}
+	z4 := base.Z{Message: m4, Score: m4.CompletedAt + m4.Retention}
+
+	inspector := NewInspector(getRedisConnOpt(t))
+
+	tests := []struct {
+		completed     map[string][]base.Z
+		qname         string
+		want          int
+		wantCompleted map[string][]base.Z
+	}{
+		{
+			completed: map[string][]base.Z{
+				"default": {z1, z2, z3},
+				"custom":  {z4},
+			},
+			qname: "default",
+			want:  3,
+			wantCompleted: map[string][]base.Z{
+				"default": {},
+				"custom":  {z4},
+			},
+		},
+		{
+			completed: map[string][]base.Z{
+				"default": {},
+			},
+			qname: "default",
+			want:  0,
+			wantCompleted: map[string][]base.Z{
+				"default": {},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		h.FlushDB(t, r)
+		h.SeedAllCompletedQueues(t, r, tc.completed)
+
+		got, err := inspector.DeleteAllCompletedTasks(tc.qname)
+		if err != nil {
+			t.Errorf("DeleteAllCompletedTasks(%q) returned error: %v", tc.qname, err)
+			continue
+		}
+		if got != tc.want {
+			t.Errorf("DeleteAllCompletedTasks(%q) = %d, want %d", tc.qname, got, tc.want)
+		}
+		for qname, want := range tc.wantCompleted {
+			gotCompleted := h.GetCompletedEntries(t, r, qname)
+			if diff := cmp.Diff(want, gotCompleted, h.SortZSetEntryOpt); diff != "" {
+				t.Errorf("unexpected completed tasks in queue %q: (-want, +got)\n%s", qname, diff)
+			}
+		}
+	}
+}
+
 func TestInspectorArchiveAllPendingTasks(t *testing.T) {
 	r := setup(t)
 	defer r.Close()
@@ -3034,6 +3196,7 @@ func TestParseOption(t *testing.T) {
 		{`Unique(1h)`, UniqueOpt, 1 * time.Hour},
 		{ProcessAt(oneHourFromNow).String(), ProcessAtOpt, oneHourFromNow},
 		{`ProcessIn(10m)`, ProcessInOpt, 10 * time.Minute},
+		{`Retention(24h)`, RetentionOpt, 24 * time.Hour},
 	}

 	for _, tc := range tests {
@@ -3065,7 +3228,7 @@ func TestParseOption(t *testing.T) {
 				if gotVal != tc.wantVal.(int) {
 					t.Fatalf("got value %v, want %v", gotVal, tc.wantVal)
 				}
-			case TimeoutOpt, UniqueOpt, ProcessInOpt:
+			case TimeoutOpt, UniqueOpt, ProcessInOpt, RetentionOpt:
 				gotVal, ok := got.Value().(time.Duration)
 				if !ok {
 					t.Fatal("returned Option with non duration value")