2
0
mirror of https://github.com/hibiken/asynq.git synced 2024-11-10 11:31:58 +08:00

Rename manager to processor

This commit is contained in:
Ken Hibino 2019-11-20 20:08:03 -08:00
parent dd0b0b358c
commit 2dd5f2c5ab
2 changed files with 31 additions and 32 deletions

View File

@ -7,15 +7,14 @@ import (
"github.com/go-redis/redis/v7" "github.com/go-redis/redis/v7"
) )
// Launcher starts the manager and poller. // Launcher starts the processor and poller.
type Launcher struct { type Launcher struct {
// running indicates whether manager and poller are both running. // running indicates whether processor and poller are both running.
running bool running bool
mu sync.Mutex mu sync.Mutex
poller *poller poller *poller
processor *processor
manager *manager
} }
// NewLauncher creates and returns a new Launcher. // NewLauncher creates and returns a new Launcher.
@ -23,17 +22,17 @@ func NewLauncher(poolSize int, opt *RedisOpt) *Launcher {
client := redis.NewClient(&redis.Options{Addr: opt.Addr, Password: opt.Password}) client := redis.NewClient(&redis.Options{Addr: opt.Addr, Password: opt.Password})
rdb := newRDB(client) rdb := newRDB(client)
poller := newPoller(rdb, 5*time.Second, []string{scheduled, retry}) poller := newPoller(rdb, 5*time.Second, []string{scheduled, retry})
manager := newManager(rdb, poolSize, nil) processor := newProcessor(rdb, poolSize, nil)
return &Launcher{ return &Launcher{
poller: poller, poller: poller,
manager: manager, processor: processor,
} }
} }
// TaskHandler handles a given task and report any error. // TaskHandler handles a given task and report any error.
type TaskHandler func(*Task) error type TaskHandler func(*Task) error
// Start starts the manager and poller. // Start starts the processor and poller.
func (l *Launcher) Start(handler TaskHandler) { func (l *Launcher) Start(handler TaskHandler) {
l.mu.Lock() l.mu.Lock()
defer l.mu.Unlock() defer l.mu.Unlock()
@ -41,13 +40,13 @@ func (l *Launcher) Start(handler TaskHandler) {
return return
} }
l.running = true l.running = true
l.manager.handler = handler l.processor.handler = handler
l.poller.start() l.poller.start()
l.manager.start() l.processor.start()
} }
// Stop stops both manager and poller. // Stop stops both processor and poller.
func (l *Launcher) Stop() { func (l *Launcher) Stop() {
l.mu.Lock() l.mu.Lock()
defer l.mu.Unlock() defer l.mu.Unlock()
@ -57,5 +56,5 @@ func (l *Launcher) Stop() {
l.running = false l.running = false
l.poller.terminate() l.poller.terminate()
l.manager.terminate() l.processor.terminate()
} }

View File

@ -8,7 +8,7 @@ import (
"time" "time"
) )
type manager struct { type processor struct {
rdb *rdb rdb *rdb
handler TaskHandler handler TaskHandler
@ -17,12 +17,12 @@ type manager struct {
// does not exceed the limit // does not exceed the limit
sema chan struct{} sema chan struct{}
// channel to communicate back to the long running "manager" goroutine. // channel to communicate back to the long running "processor" goroutine.
done chan struct{} done chan struct{}
} }
func newManager(rdb *rdb, numWorkers int, handler TaskHandler) *manager { func newProcessor(rdb *rdb, numWorkers int, handler TaskHandler) *processor {
return &manager{ return &processor{
rdb: rdb, rdb: rdb,
handler: handler, handler: handler,
sema: make(chan struct{}, numWorkers), sema: make(chan struct{}, numWorkers),
@ -30,40 +30,40 @@ func newManager(rdb *rdb, numWorkers int, handler TaskHandler) *manager {
} }
} }
func (m *manager) terminate() { func (p *processor) terminate() {
// send a signal to the manager goroutine to stop // send a signal to the processor goroutine to stop
// processing tasks from the queue. // processing tasks from the queue.
m.done <- struct{}{} p.done <- struct{}{}
fmt.Println("--- Waiting for all workers to finish ---") fmt.Println("--- Waiting for all workers to finish ---")
for i := 0; i < cap(m.sema); i++ { for i := 0; i < cap(p.sema); i++ {
// block until all workers have released the token // block until all workers have released the token
m.sema <- struct{}{} p.sema <- struct{}{}
} }
fmt.Println("--- All workers have finished! ----") fmt.Println("--- All workers have finished! ----")
} }
func (m *manager) start() { func (p *processor) start() {
go func() { go func() {
for { for {
select { select {
case <-m.done: case <-p.done:
fmt.Println("-------------[Manager]---------------") fmt.Println("-------------[Processor]---------------")
fmt.Println("Manager shutting down...") fmt.Println("Processor shutting down...")
fmt.Println("-------------------------------------") fmt.Println("-------------------------------------")
return return
default: default:
m.processTasks() p.exec()
} }
} }
}() }()
} }
func (m *manager) processTasks() { func (p *processor) exec() {
// pull message out of the queue and process it // pull message out of the queue and process it
// TODO(hibiken): sort the list of queues in order of priority // TODO(hibiken): sort the list of queues in order of priority
// NOTE: BLPOP needs to timeout in case a new queue is added. // NOTE: BLPOP needs to timeout in case a new queue is added.
msg, err := m.rdb.bpop(5*time.Second, m.rdb.listQueues()...) msg, err := p.rdb.bpop(5*time.Second, p.rdb.listQueues()...)
if err != nil { if err != nil {
switch err { switch err {
case errQueuePopTimeout: case errQueuePopTimeout:
@ -79,14 +79,14 @@ func (m *manager) processTasks() {
} }
t := &Task{Type: msg.Type, Payload: msg.Payload} t := &Task{Type: msg.Type, Payload: msg.Payload}
m.sema <- struct{}{} // acquire token p.sema <- struct{}{} // acquire token
go func(task *Task) { go func(task *Task) {
defer func() { <-m.sema }() // release token defer func() { <-p.sema }() // release token
err := m.handler(task) err := p.handler(task)
if err != nil { if err != nil {
if msg.Retried >= msg.Retry { if msg.Retried >= msg.Retry {
fmt.Println("Retry exhausted!!!") fmt.Println("Retry exhausted!!!")
if err := m.rdb.kill(msg); err != nil { if err := p.rdb.kill(msg); err != nil {
log.Printf("[SERVER ERROR] could not add task %+v to 'dead' set\n", err) log.Printf("[SERVER ERROR] could not add task %+v to 'dead' set\n", err)
} }
return return
@ -96,7 +96,7 @@ func (m *manager) processTasks() {
fmt.Printf("[DEBUG] retying the task in %v\n", retryAt.Sub(time.Now())) fmt.Printf("[DEBUG] retying the task in %v\n", retryAt.Sub(time.Now()))
msg.Retried++ msg.Retried++
msg.ErrorMsg = err.Error() msg.ErrorMsg = err.Error()
if err := m.rdb.zadd(retry, float64(retryAt.Unix()), msg); err != nil { if err := p.rdb.zadd(retry, float64(retryAt.Unix()), msg); err != nil {
// TODO(hibiken): Not sure how to handle this error // TODO(hibiken): Not sure how to handle this error
log.Printf("[SEVERE ERROR] could not add msg %+v to 'retry' set: %v\n", msg, err) log.Printf("[SEVERE ERROR] could not add msg %+v to 'retry' set: %v\n", msg, err)
return return