1 import { AsyncQueue, forever, queue } from 'async'
2 import * as Sequelize from 'sequelize'
3 import { JobCategory } from '../../../shared'
4 import { logger } from '../../helpers/logger'
5 import { JOB_STATES, JOBS_FETCH_LIMIT_PER_CYCLE, JOBS_FETCHING_INTERVAL } from '../../initializers'
6 import { JobModel } from '../../models/job/job'
8 export interface JobHandler<P, T> {
9 process (data: object, jobId: number): Promise<T>
10 onError (err: Error, jobId: number)
11 onSuccess (jobId: number, jobResult: T, jobScheduler: JobScheduler<P, T>): Promise<any>
13 type JobQueueCallback = (err: Error) => void
15 class JobScheduler<P, T> {
18 private jobCategory: JobCategory,
19 private jobHandlers: { [ id: string ]: JobHandler<P, T> }
23 const limit = JOBS_FETCH_LIMIT_PER_CYCLE[this.jobCategory]
25 logger.info('Jobs scheduler %s activated.', this.jobCategory)
27 const jobsQueue = queue<JobModel, JobQueueCallback>(this.processJob.bind(this))
29 // Finish processing jobs from a previous start
30 const state = JOB_STATES.PROCESSING
32 const jobs = await JobModel.listWithLimitByCategory(limit, state, this.jobCategory)
34 this.enqueueJobs(jobsQueue, jobs)
36 logger.error('Cannot list pending jobs.', err)
41 if (jobsQueue.length() !== 0) {
42 // Finish processing the queue first
43 return setTimeout(next, JOBS_FETCHING_INTERVAL)
46 const state = JOB_STATES.PENDING
48 const jobs = await JobModel.listWithLimitByCategory(limit, state, this.jobCategory)
50 this.enqueueJobs(jobsQueue, jobs)
52 logger.error('Cannot list pending jobs.', err)
55 // Optimization: we could use "drain" from queue object
56 return setTimeout(next, JOBS_FETCHING_INTERVAL)
59 err => logger.error('Error in job scheduler queue.', err)
63 createJob (transaction: Sequelize.Transaction, handlerName: string, handlerInputData: P) {
65 state: JOB_STATES.PENDING,
66 category: this.jobCategory,
71 const options = { transaction }
73 return JobModel.create(createQuery, options)
76 private enqueueJobs (jobsQueue: AsyncQueue<JobModel>, jobs: JobModel[]) {
77 jobs.forEach(job => jobsQueue.push(job))
80 private async processJob (job: JobModel, callback: (err: Error) => void) {
81 const jobHandler = this.jobHandlers[job.handlerName]
82 if (jobHandler === undefined) {
83 const errorString = 'Unknown job handler ' + job.handlerName + ' for job ' + job.id
84 logger.error(errorString)
86 const error = new Error(errorString)
87 await this.onJobError(jobHandler, job, error)
88 return callback(error)
91 logger.info('Processing job %d with handler %s.', job.id, job.handlerName)
93 job.state = JOB_STATES.PROCESSING
97 const result: T = await jobHandler.process(job.handlerInputData, job.id)
98 await this.onJobSuccess(jobHandler, job, result)
100 logger.error('Error in job handler %s.', job.handlerName, err)
103 await this.onJobError(jobHandler, job, err)
105 this.cannotSaveJobError(innerErr)
106 return callback(innerErr)
110 return callback(null)
113 private async onJobError (jobHandler: JobHandler<P, T>, job: JobModel, err: Error) {
114 job.state = JOB_STATES.ERROR
118 if (jobHandler) await jobHandler.onError(err, job.id)
120 this.cannotSaveJobError(err)
124 private async onJobSuccess (jobHandler: JobHandler<P, T>, job: JobModel, jobResult: T) {
125 job.state = JOB_STATES.SUCCESS
129 await jobHandler.onSuccess(job.id, jobResult, this)
131 this.cannotSaveJobError(err)
135 private cannotSaveJobError (err: Error) {
136 logger.error('Cannot save new job state.', err)
140 // ---------------------------------------------------------------------------