pkg/asynq
2
0
Fork 0
mirror of https://github.com/hibiken/asynq.git synced 2025-10-26 11:16:12 +08:00
Code Issues Projects Releases Wiki Activity

Introduce Task Results

Browse Source 
* Added Retention Option to specify retention TTL for tasks
* Added ResultWriter as a client interface to write result data for the associated task
This commit is contained in:
Ken Hibino
2021-11-05 16:52:54 -07:00
parent 4638405cbd
commit f4ddac4dcc
33 changed files with 2099 additions and 846 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
processor.go
View File

@@ -201,14 +201,24 @@ func (p *processor) exec() {
select {
case <-ctx.Done():
// already canceled (e.g. deadline exceeded).
p.retryOrArchive(ctx, msg, ctx.Err())
p.handleFailedMessage(ctx, msg, ctx.Err())
return
default:
}
resCh := make(chan error, 1)
go func() {
resCh <- p.perform(ctx, NewTask(msg.Type, msg.Payload))
task := newTask(
msg.Type,
msg.Payload,
&ResultWriter{
id: msg.ID,
qname: msg.Queue,
broker: p.broker,
ctx: ctx,
},
)
resCh <- p.perform(ctx, task)
}()
select {
@@ -218,18 +228,14 @@ func (p *processor) exec() {
p.requeue(msg)
return
case <-ctx.Done():
p.retryOrArchive(ctx, msg, ctx.Err())
p.handleFailedMessage(ctx, msg, ctx.Err())
return
case resErr := <-resCh:
// Note: One of three things should happen.
// 1) Done -> Removes the message from Active
// 2) Retry -> Removes the message from Active & Adds the message to Retry
// 3) Archive -> Removes the message from Active & Adds the message to archive
if resErr != nil {
p.retryOrArchive(ctx, msg, resErr)
p.handleFailedMessage(ctx, msg, resErr)
return
}
p.markAsDone(ctx, msg)
p.handleSucceededMessage(ctx, msg)
}
}()
}
@@ -244,6 +250,34 @@ func (p *processor) requeue(msg *base.TaskMessage) {
}
}
func (p *processor) handleSucceededMessage(ctx context.Context, msg *base.TaskMessage) {
if msg.Retention > 0 {
p.markAsComplete(ctx, msg)
} else {
p.markAsDone(ctx, msg)
}
}
func (p *processor) markAsComplete(ctx context.Context, msg *base.TaskMessage) {
err := p.broker.MarkAsComplete(msg)
if err != nil {
errMsg := fmt.Sprintf("Could not move task id=%s type=%q from %q to %q: %+v",
msg.ID, msg.Type, base.ActiveKey(msg.Queue), base.CompletedKey(msg.Queue), err)
deadline, ok := ctx.Deadline()
if !ok {
panic("asynq: internal error: missing deadline in context")
}
p.logger.Warnf("%s; Will retry syncing", errMsg)
p.syncRequestCh <- &syncRequest{
fn: func() error {
return p.broker.MarkAsComplete(msg)
},
errMsg: errMsg,
deadline: deadline,
}
}
}
func (p *processor) markAsDone(ctx context.Context, msg *base.TaskMessage) {
err := p.broker.Done(msg)
if err != nil {
@@ -267,7 +301,7 @@ func (p *processor) markAsDone(ctx context.Context, msg *base.TaskMessage) {
// the task should not be retried and should be archived instead.
var SkipRetry = errors.New("skip retry for the task")
func (p *processor) retryOrArchive(ctx context.Context, msg *base.TaskMessage, err error) {
func (p *processor) handleFailedMessage(ctx context.Context, msg *base.TaskMessage, err error) {
if p.errHandler != nil {
p.errHandler.HandleError(ctx, NewTask(msg.Type, msg.Payload), err)
}