mirror of
https://github.com/hibiken/asynq.git
synced 2024-12-25 07:12:17 +08:00
Rename manager to processor
This commit is contained in:
parent
dd0b0b358c
commit
2dd5f2c5ab
21
launcher.go
21
launcher.go
@ -7,15 +7,14 @@ import (
|
|||||||
"github.com/go-redis/redis/v7"
|
"github.com/go-redis/redis/v7"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Launcher starts the manager and poller.
|
// Launcher starts the processor and poller.
|
||||||
type Launcher struct {
|
type Launcher struct {
|
||||||
// running indicates whether manager and poller are both running.
|
// running indicates whether processor and poller are both running.
|
||||||
running bool
|
running bool
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
|
|
||||||
poller *poller
|
poller *poller
|
||||||
|
processor *processor
|
||||||
manager *manager
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewLauncher creates and returns a new Launcher.
|
// NewLauncher creates and returns a new Launcher.
|
||||||
@ -23,17 +22,17 @@ func NewLauncher(poolSize int, opt *RedisOpt) *Launcher {
|
|||||||
client := redis.NewClient(&redis.Options{Addr: opt.Addr, Password: opt.Password})
|
client := redis.NewClient(&redis.Options{Addr: opt.Addr, Password: opt.Password})
|
||||||
rdb := newRDB(client)
|
rdb := newRDB(client)
|
||||||
poller := newPoller(rdb, 5*time.Second, []string{scheduled, retry})
|
poller := newPoller(rdb, 5*time.Second, []string{scheduled, retry})
|
||||||
manager := newManager(rdb, poolSize, nil)
|
processor := newProcessor(rdb, poolSize, nil)
|
||||||
return &Launcher{
|
return &Launcher{
|
||||||
poller: poller,
|
poller: poller,
|
||||||
manager: manager,
|
processor: processor,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TaskHandler handles a given task and report any error.
|
// TaskHandler handles a given task and report any error.
|
||||||
type TaskHandler func(*Task) error
|
type TaskHandler func(*Task) error
|
||||||
|
|
||||||
// Start starts the manager and poller.
|
// Start starts the processor and poller.
|
||||||
func (l *Launcher) Start(handler TaskHandler) {
|
func (l *Launcher) Start(handler TaskHandler) {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
@ -41,13 +40,13 @@ func (l *Launcher) Start(handler TaskHandler) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
l.running = true
|
l.running = true
|
||||||
l.manager.handler = handler
|
l.processor.handler = handler
|
||||||
|
|
||||||
l.poller.start()
|
l.poller.start()
|
||||||
l.manager.start()
|
l.processor.start()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops both manager and poller.
|
// Stop stops both processor and poller.
|
||||||
func (l *Launcher) Stop() {
|
func (l *Launcher) Stop() {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
@ -57,5 +56,5 @@ func (l *Launcher) Stop() {
|
|||||||
l.running = false
|
l.running = false
|
||||||
|
|
||||||
l.poller.terminate()
|
l.poller.terminate()
|
||||||
l.manager.terminate()
|
l.processor.terminate()
|
||||||
}
|
}
|
||||||
|
@ -8,7 +8,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
type manager struct {
|
type processor struct {
|
||||||
rdb *rdb
|
rdb *rdb
|
||||||
|
|
||||||
handler TaskHandler
|
handler TaskHandler
|
||||||
@ -17,12 +17,12 @@ type manager struct {
|
|||||||
// does not exceed the limit
|
// does not exceed the limit
|
||||||
sema chan struct{}
|
sema chan struct{}
|
||||||
|
|
||||||
// channel to communicate back to the long running "manager" goroutine.
|
// channel to communicate back to the long running "processor" goroutine.
|
||||||
done chan struct{}
|
done chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newManager(rdb *rdb, numWorkers int, handler TaskHandler) *manager {
|
func newProcessor(rdb *rdb, numWorkers int, handler TaskHandler) *processor {
|
||||||
return &manager{
|
return &processor{
|
||||||
rdb: rdb,
|
rdb: rdb,
|
||||||
handler: handler,
|
handler: handler,
|
||||||
sema: make(chan struct{}, numWorkers),
|
sema: make(chan struct{}, numWorkers),
|
||||||
@ -30,40 +30,40 @@ func newManager(rdb *rdb, numWorkers int, handler TaskHandler) *manager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manager) terminate() {
|
func (p *processor) terminate() {
|
||||||
// send a signal to the manager goroutine to stop
|
// send a signal to the processor goroutine to stop
|
||||||
// processing tasks from the queue.
|
// processing tasks from the queue.
|
||||||
m.done <- struct{}{}
|
p.done <- struct{}{}
|
||||||
|
|
||||||
fmt.Println("--- Waiting for all workers to finish ---")
|
fmt.Println("--- Waiting for all workers to finish ---")
|
||||||
for i := 0; i < cap(m.sema); i++ {
|
for i := 0; i < cap(p.sema); i++ {
|
||||||
// block until all workers have released the token
|
// block until all workers have released the token
|
||||||
m.sema <- struct{}{}
|
p.sema <- struct{}{}
|
||||||
}
|
}
|
||||||
fmt.Println("--- All workers have finished! ----")
|
fmt.Println("--- All workers have finished! ----")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manager) start() {
|
func (p *processor) start() {
|
||||||
go func() {
|
go func() {
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-m.done:
|
case <-p.done:
|
||||||
fmt.Println("-------------[Manager]---------------")
|
fmt.Println("-------------[Processor]---------------")
|
||||||
fmt.Println("Manager shutting down...")
|
fmt.Println("Processor shutting down...")
|
||||||
fmt.Println("-------------------------------------")
|
fmt.Println("-------------------------------------")
|
||||||
return
|
return
|
||||||
default:
|
default:
|
||||||
m.processTasks()
|
p.exec()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manager) processTasks() {
|
func (p *processor) exec() {
|
||||||
// pull message out of the queue and process it
|
// pull message out of the queue and process it
|
||||||
// TODO(hibiken): sort the list of queues in order of priority
|
// TODO(hibiken): sort the list of queues in order of priority
|
||||||
// NOTE: BLPOP needs to timeout in case a new queue is added.
|
// NOTE: BLPOP needs to timeout in case a new queue is added.
|
||||||
msg, err := m.rdb.bpop(5*time.Second, m.rdb.listQueues()...)
|
msg, err := p.rdb.bpop(5*time.Second, p.rdb.listQueues()...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
switch err {
|
switch err {
|
||||||
case errQueuePopTimeout:
|
case errQueuePopTimeout:
|
||||||
@ -79,14 +79,14 @@ func (m *manager) processTasks() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
t := &Task{Type: msg.Type, Payload: msg.Payload}
|
t := &Task{Type: msg.Type, Payload: msg.Payload}
|
||||||
m.sema <- struct{}{} // acquire token
|
p.sema <- struct{}{} // acquire token
|
||||||
go func(task *Task) {
|
go func(task *Task) {
|
||||||
defer func() { <-m.sema }() // release token
|
defer func() { <-p.sema }() // release token
|
||||||
err := m.handler(task)
|
err := p.handler(task)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if msg.Retried >= msg.Retry {
|
if msg.Retried >= msg.Retry {
|
||||||
fmt.Println("Retry exhausted!!!")
|
fmt.Println("Retry exhausted!!!")
|
||||||
if err := m.rdb.kill(msg); err != nil {
|
if err := p.rdb.kill(msg); err != nil {
|
||||||
log.Printf("[SERVER ERROR] could not add task %+v to 'dead' set\n", err)
|
log.Printf("[SERVER ERROR] could not add task %+v to 'dead' set\n", err)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@ -96,7 +96,7 @@ func (m *manager) processTasks() {
|
|||||||
fmt.Printf("[DEBUG] retying the task in %v\n", retryAt.Sub(time.Now()))
|
fmt.Printf("[DEBUG] retying the task in %v\n", retryAt.Sub(time.Now()))
|
||||||
msg.Retried++
|
msg.Retried++
|
||||||
msg.ErrorMsg = err.Error()
|
msg.ErrorMsg = err.Error()
|
||||||
if err := m.rdb.zadd(retry, float64(retryAt.Unix()), msg); err != nil {
|
if err := p.rdb.zadd(retry, float64(retryAt.Unix()), msg); err != nil {
|
||||||
// TODO(hibiken): Not sure how to handle this error
|
// TODO(hibiken): Not sure how to handle this error
|
||||||
log.Printf("[SEVERE ERROR] could not add msg %+v to 'retry' set: %v\n", msg, err)
|
log.Printf("[SEVERE ERROR] could not add msg %+v to 'retry' set: %v\n", msg, err)
|
||||||
return
|
return
|
Loading…
Reference in New Issue
Block a user