go-pkg/asynq
2
0
Fork 0
mirror of https://github.com/hibiken/asynq.git synced 2025-04-22 08:40:22 +08:00
Code Issues Projects Releases Wiki Activity

Use protobuf to encode task message

Browse Source
This commit is contained in:
Ken Hibino 2021-03-01 21:33:36 -08:00
parent 47543b36fd
commit 4046932fde
7 changed files with 60 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@ -4,6 +4,7 @@ go 1.13
require (
github.com/go-redis/redis/v7 v7.4.0
github.com/golang/protobuf v1.4.1
github.com/google/go-cmp v0.5.0
github.com/google/uuid v1.2.0
github.com/robfig/cron/v3 v3.0.1
@ -11,6 +12,6 @@ require (
go.uber.org/goleak v0.10.0
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
google.golang.org/protobuf v1.25.0 // indirect
google.golang.org/protobuf v1.25.0
gopkg.in/yaml.v2 v2.2.7 // indirect
)

2
go.sum
View File

@ -22,12 +22,14 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

8
internal/asynqtest/asynqtest.go
View File

@ -6,7 +6,6 @@
package asynqtest
import (
"encoding/json"
"math"
"sort"
"testing"
@ -130,7 +129,7 @@ func TaskMessageWithError(t base.TaskMessage, errMsg string) *base.TaskMessage {
// Calling test will fail if marshaling errors out.
func MustMarshal(tb testing.TB, msg *base.TaskMessage) string {
tb.Helper()
data, err := json.Marshal(msg)
data, err := base.EncodeMessage(msg)
if err != nil {
tb.Fatal(err)
}
@ -141,12 +140,11 @@ func MustMarshal(tb testing.TB, msg *base.TaskMessage) string {
// Calling test will fail if unmarshaling errors out.
func MustUnmarshal(tb testing.TB, data string) *base.TaskMessage {
tb.Helper()
var msg base.TaskMessage
err := json.Unmarshal([]byte(data), &msg)
msg, err := base.DecodeMessage(data)
if err != nil {
tb.Fatal(err)
}
return &msg
return msg
}
// MustMarshalSlice marshals a slice of task messages and return a slice of

48
internal/base/base.go
View File

@ -6,6 +6,7 @@
package base
import (
"bytes"
"context"
"encoding/json"
"fmt"
@ -16,6 +17,8 @@ import (
"github.com/go-redis/redis/v7"
"github.com/google/uuid"
pb "github.com/hibiken/asynq/internal/proto"
"google.golang.org/protobuf/proto"
)
// Version of asynq library and CLI.
@ -195,8 +198,25 @@ type TaskMessage struct {
}
// EncodeMessage marshals the given task message in JSON and returns an encoded string.
// TODO: Should return []byte instead of string
func EncodeMessage(msg *TaskMessage) (string, error) {
b, err := json.Marshal(msg)
payload, err := json.Marshal(msg.Payload)
if err != nil {
return "", err
}
pbmsg := pb.TaskMessage{
Type: msg.Type,
Payload: payload,
Id: msg.ID.String(),
Queue: msg.Queue,
Retry: int32(msg.Retry),
Retried: int32(msg.Retried),
ErrorMsg: msg.ErrorMsg,
Timeout: msg.Timeout,
Deadline: msg.Deadline,
UniqueKey: msg.UniqueKey,
}
b, err := proto.Marshal(&pbmsg)
if err != nil {
return "", err
}
@ -204,14 +224,30 @@ func EncodeMessage(msg *TaskMessage) (string, error) {
}
// DecodeMessage unmarshals the given encoded string and returns a decoded task message.
// TODO: should take []byte instead of string
func DecodeMessage(s string) (*TaskMessage, error) {
d := json.NewDecoder(strings.NewReader(s))
d.UseNumber()
var msg TaskMessage
if err := d.Decode(&msg); err != nil {
var pbmsg pb.TaskMessage
if err := proto.Unmarshal([]byte(s), &pbmsg); err != nil {
return nil, err
}
return &msg, nil
d := json.NewDecoder(bytes.NewReader(pbmsg.Payload))
d.UseNumber()
payload := make(map[string]interface{})
if err := d.Decode(&payload); err != nil {
return nil, err
}
return &TaskMessage{
Type: pbmsg.Type,
Payload: payload,
ID: uuid.MustParse(pbmsg.Id),
Queue: pbmsg.Queue,
Retry: int(pbmsg.Retry),
Retried: int(pbmsg.Retried),
ErrorMsg: pbmsg.ErrorMsg,
Timeout: pbmsg.Timeout,
Deadline: pbmsg.Deadline,
UniqueKey: pbmsg.UniqueKey,
}, nil
}
// Z represents sorted set member.

16
internal/proto/asynq.pb.go
View File

@ -34,8 +34,8 @@ type TaskMessage struct {
Payload []byte `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"`
Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"`
Queue string `protobuf:"bytes,4,opt,name=queue,proto3" json:"queue,omitempty"`
Retry string `protobuf:"bytes,5,opt,name=retry,proto3" json:"retry,omitempty"`
Retried string `protobuf:"bytes,6,opt,name=retried,proto3" json:"retried,omitempty"`
Retry int32 `protobuf:"varint,5,opt,name=retry,proto3" json:"retry,omitempty"`
Retried int32 `protobuf:"varint,6,opt,name=retried,proto3" json:"retried,omitempty"`
ErrorMsg string `protobuf:"bytes,7,opt,name=error_msg,json=errorMsg,proto3" json:"error_msg,omitempty"`
Timeout int64 `protobuf:"varint,8,opt,name=timeout,proto3" json:"timeout,omitempty"`
Deadline int64 `protobuf:"varint,9,opt,name=deadline,proto3" json:"deadline,omitempty"`
@ -102,18 +102,18 @@ func (x *TaskMessage) GetQueue() string {
return ""
}
func (x *TaskMessage) GetRetry() string {
func (x *TaskMessage) GetRetry() int32 {
if x != nil {
return x.Retry
}
return ""
return 0
}
func (x *TaskMessage) GetRetried() string {
func (x *TaskMessage) GetRetried() int32 {
if x != nil {
return x.Retried
}
return ""
return 0
}
func (x *TaskMessage) GetErrorMsg() string {
@ -155,9 +155,9 @@ var file_asynq_proto_rawDesc = []byte{
0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28,
0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x75, 0x65, 0x18, 0x04,
0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x75, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72,
0x65, 0x74, 0x72, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x65, 0x74, 0x72,
0x65, 0x74, 0x72, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x72, 0x65, 0x74, 0x72,
0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01,
0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x65,
0x28, 0x05, 0x52, 0x07, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x65,
0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08,
0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65,
0x6f, 0x75, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f,

4
internal/proto/asynq.proto
View File

@ -14,9 +14,9 @@ message TaskMessage {
string queue = 4;
string retry = 5;
int32 retry = 5;
string retried = 6;
int32 retried = 6;
string error_msg = 7;

2
internal/rdb/rdb_test.go
View File

@ -101,7 +101,7 @@ func TestEnqueueUnique(t *testing.T) {
m1 := base.TaskMessage{
ID: uuid.New(),
Type: "email",
Payload: map[string]interface{}{"user_id": float64(123)},
Payload: map[string]interface{}{"user_id": json.Number("123")},
Queue: base.DefaultQueueName,
UniqueKey: base.UniqueKey(base.DefaultQueueName, "email", map[string]interface{}{"user_id": 123}),
}