Add tasks processed/failed and error_rate charts

metrics_handler.go

@@ -15,6 +15,9 @@ type getMetricsResponse struct {
 	QueueSize            *json.RawMessage `json:"queue_size"`
 	QueueLatency         *json.RawMessage `json:"queue_latency_seconds"`
 	QueueMemUsgApprox    *json.RawMessage `json:"queue_memory_usage_approx_bytes"`
+	ProcessedPerSecond   *json.RawMessage `json:"tasks_processed_per_second"`
+	FailedPerSecond      *json.RawMessage `json:"tasks_failed_per_second"`
+	ErrorRate            *json.RawMessage `json:"error_rate"`
 	PendingTasksByQueue  *json.RawMessage `json:"pending_tasks_by_queue"`
 	RetryTasksByQueue    *json.RawMessage `json:"retry_tasks_by_queue"`
 	ArchivedTasksByQueue *json.RawMessage `json:"archived_tasks_by_queue"`
@@ -36,6 +39,19 @@ func newGetMetricsHandlerFunc(client *http.Client, prometheusAddr string) http.H
 		err   error
 	}
 
+	// List of PromQLs.
+	const (
+		promQLQueueSize      = "asynq_queue_size"
+		promQLQueueLatency   = "asynq_queue_latency_seconds"
+		promQLMemUsage       = "asynq_queue_memory_usage_approx_bytes"
+		promQLProcessedTasks = "rate(asynq_tasks_processed_total[5m])"
+		promQLFailedTasks    = "rate(asynq_tasks_failed_total[5m])"
+		promQLErrorRate      = "rate(asynq_tasks_failed_total[5m]) / rate(asynq_tasks_processed_total[5m])"
+		promQLPendingTasks   = "asynq_tasks_enqueued_total{state=\"pending\"}"
+		promQLRetryTasks     = "asynq_tasks_enqueued_total{state=\"retry\"}"
+		promQLArchivedTasks  = "asynq_tasks_enqueued_total{state=\"archived\"}"
+	)
+
 	// Optional query params:
 	// `duration_sec`: specifies the number of seconds to scan
 	// `end_time`:     specifies the end_time in Unix time seconds
@@ -47,12 +63,15 @@ func newGetMetricsHandlerFunc(client *http.Client, prometheusAddr string) http.H
 		}
 		// List of queries (i.e. promQL) to send to prometheus server.
 		queries := []string{
-			"asynq_queue_size",
+			promQLQueueSize,
-			"asynq_queue_latency_seconds",
+			promQLQueueLatency,
-			"asynq_queue_memory_usage_approx_bytes",
+			promQLMemUsage,
-			"asynq_tasks_enqueued_total{state=\"pending\"}",
+			promQLProcessedTasks,
-			"asynq_tasks_enqueued_total{state=\"retry\"}",
+			promQLFailedTasks,
-			"asynq_tasks_enqueued_total{state=\"archived\"}",
+			promQLErrorRate,
+			promQLPendingTasks,
+			promQLRetryTasks,
+			promQLArchivedTasks,
 		}
 		resp := getMetricsResponse{}
 		// Make multiple API calls concurrently
@@ -72,17 +91,23 @@ func newGetMetricsHandlerFunc(client *http.Client, prometheusAddr string) http.H
 				return
 			}
 			switch r.query {
-			case "asynq_queue_size":
+			case promQLQueueSize:
 				resp.QueueSize = r.msg
-			case "asynq_queue_latency_seconds":
+			case promQLQueueLatency:
 				resp.QueueLatency = r.msg
-			case "asynq_queue_memory_usage_approx_bytes":
+			case promQLMemUsage:
 				resp.QueueMemUsgApprox = r.msg
-			case "asynq_tasks_enqueued_total{state=\"pending\"}":
+			case promQLProcessedTasks:
+				resp.ProcessedPerSecond = r.msg
+			case promQLFailedTasks:
+				resp.FailedPerSecond = r.msg
+			case promQLErrorRate:
+				resp.ErrorRate = r.msg
+			case promQLPendingTasks:
 				resp.PendingTasksByQueue = r.msg
-			case "asynq_tasks_enqueued_total{state=\"retry\"}":
+			case promQLRetryTasks:
 				resp.RetryTasksByQueue = r.msg
-			case "asynq_tasks_enqueued_total{state=\"archived\"}":
+			case promQLArchivedTasks:
 				resp.ArchivedTasksByQueue = r.msg
 			}
 			if n == 0 {

ui/src/api.ts

@@ -80,6 +80,9 @@ export interface MetricsResponse {
   queue_size: PrometheusMetricsResponse;
   queue_latency_seconds: PrometheusMetricsResponse;
   queue_memory_usage_approx_bytes: PrometheusMetricsResponse;
+  tasks_processed_per_second: PrometheusMetricsResponse;
+  tasks_failed_per_second: PrometheusMetricsResponse;
+  error_rate: PrometheusMetricsResponse;
   pending_tasks_by_queue: PrometheusMetricsResponse;
   retry_tasks_by_queue: PrometheusMetricsResponse;
   archived_tasks_by_queue: PrometheusMetricsResponse;

ui/src/views/MetricsView.tsx

@@ -126,6 +126,36 @@ function MetricsView(props: Props) {
         />
       </div>
       <Grid container spacing={3}>
+        {data?.tasks_processed_per_second && (
+          <Grid item xs={12}>
+            <ChartRow
+              title="Tasks Processed"
+              metrics={data.tasks_processed_per_second}
+              endTime={endTimeSec}
+              startTime={endTimeSec - durationSec}
+            />
+          </Grid>
+        )}
+        {data?.tasks_failed_per_second && (
+          <Grid item xs={12}>
+            <ChartRow
+              title="Tasks Failed"
+              metrics={data.tasks_failed_per_second}
+              endTime={endTimeSec}
+              startTime={endTimeSec - durationSec}
+            />
+          </Grid>
+        )}
+        {data?.error_rate && (
+          <Grid item xs={12}>
+            <ChartRow
+              title="Error Rate"
+              metrics={data.error_rate}
+              endTime={endTimeSec}
+              startTime={endTimeSec - durationSec}
+            />
+          </Grid>
+        )}
         {data?.queue_size && (
           <Grid item xs={12}>
             <ChartRow