import { AsyncQueue, forever, queue } from 'async'
import * as Sequelize from 'sequelize'
-
-import { database as db } from '../../initializers/database'
-import {
- JOBS_FETCHING_INTERVAL,
- JOBS_FETCH_LIMIT_PER_CYCLE,
- JOB_STATES
-} from '../../initializers'
+import { JobCategory } from '../../../shared'
import { logger } from '../../helpers'
-import { JobInstance } from '../../models'
-import { JobHandler, jobHandlers } from './handlers'
+import { JOB_STATES, JOBS_FETCH_LIMIT_PER_CYCLE, JOBS_FETCHING_INTERVAL } from '../../initializers'
+import { JobModel } from '../../models/job/job'
+export interface JobHandler<P, T> {
+ process (data: object, jobId: number): Promise<T>
+ onError (err: Error, jobId: number)
+ onSuccess (jobId: number, jobResult: T, jobScheduler: JobScheduler<P, T>): Promise<any>
+}
type JobQueueCallback = (err: Error) => void
-class JobScheduler {
-
- private static instance: JobScheduler
+class JobScheduler<P, T> {
- private constructor () { }
+ constructor (
+ private jobCategory: JobCategory,
+ private jobHandlers: { [ id: string ]: JobHandler<P, T> }
+ ) {}
- static get Instance () {
- return this.instance || (this.instance = new this())
- }
-
- activate () {
- const limit = JOBS_FETCH_LIMIT_PER_CYCLE
+ async activate () {
+ const limit = JOBS_FETCH_LIMIT_PER_CYCLE[this.jobCategory]
- logger.info('Jobs scheduler activated.')
+ logger.info('Jobs scheduler %s activated.', this.jobCategory)
- const jobsQueue = queue<JobInstance, JobQueueCallback>(this.processJob.bind(this))
+ const jobsQueue = queue<JobModel, JobQueueCallback>(this.processJob.bind(this))
// Finish processing jobs from a previous start
const state = JOB_STATES.PROCESSING
- db.Job.listWithLimit(limit, state)
- .then(jobs => {
- this.enqueueJobs(jobsQueue, jobs)
-
- forever(
- next => {
- if (jobsQueue.length() !== 0) {
- // Finish processing the queue first
- return setTimeout(next, JOBS_FETCHING_INTERVAL)
- }
-
- const state = JOB_STATES.PENDING
- db.Job.listWithLimit(limit, state)
- .then(jobs => {
- this.enqueueJobs(jobsQueue, jobs)
-
- // Optimization: we could use "drain" from queue object
- return setTimeout(next, JOBS_FETCHING_INTERVAL)
- })
- .catch(err => logger.error('Cannot list pending jobs.', err))
- },
-
- err => logger.error('Error in job scheduler queue.', err)
- )
- })
- .catch(err => logger.error('Cannot list pending jobs.', err))
+ try {
+ const jobs = await JobModel.listWithLimitByCategory(limit, state, this.jobCategory)
+
+ this.enqueueJobs(jobsQueue, jobs)
+ } catch (err) {
+ logger.error('Cannot list pending jobs.', err)
+ }
+
+ forever(
+ async next => {
+ if (jobsQueue.length() !== 0) {
+ // Finish processing the queue first
+ return setTimeout(next, JOBS_FETCHING_INTERVAL)
+ }
+
+ const state = JOB_STATES.PENDING
+ try {
+ const jobs = await JobModel.listWithLimitByCategory(limit, state, this.jobCategory)
+
+ this.enqueueJobs(jobsQueue, jobs)
+ } catch (err) {
+ logger.error('Cannot list pending jobs.', err)
+ }
+
+ // Optimization: we could use "drain" from queue object
+ return setTimeout(next, JOBS_FETCHING_INTERVAL)
+ },
+
+ err => logger.error('Error in job scheduler queue.', err)
+ )
}
- createJob (transaction: Sequelize.Transaction, handlerName: string, handlerInputData: object) {
+ createJob (transaction: Sequelize.Transaction, handlerName: string, handlerInputData: P) {
const createQuery = {
state: JOB_STATES.PENDING,
+ category: this.jobCategory,
handlerName,
handlerInputData
}
+
const options = { transaction }
- return db.Job.create(createQuery, options)
+ return JobModel.create(createQuery, options)
}
- private enqueueJobs (jobsQueue: AsyncQueue<JobInstance>, jobs: JobInstance[]) {
+ private enqueueJobs (jobsQueue: AsyncQueue<JobModel>, jobs: JobModel[]) {
jobs.forEach(job => jobsQueue.push(job))
}
- private processJob (job: JobInstance, callback: (err: Error) => void) {
- const jobHandler = jobHandlers[job.handlerName]
+ private async processJob (job: JobModel, callback: (err: Error) => void) {
+ const jobHandler = this.jobHandlers[job.handlerName]
if (jobHandler === undefined) {
- logger.error('Unknown job handler for job %s.', job.handlerName)
- return callback(null)
+ const errorString = 'Unknown job handler ' + job.handlerName + ' for job ' + job.id
+ logger.error(errorString)
+
+ const error = new Error(errorString)
+ await this.onJobError(jobHandler, job, error)
+ return callback(error)
}
logger.info('Processing job %d with handler %s.', job.id, job.handlerName)
job.state = JOB_STATES.PROCESSING
- return job.save()
- .then(() => {
- return jobHandler.process(job.handlerInputData, job.id)
- })
- .then(
- result => {
- return this.onJobSuccess(jobHandler, job, result)
- },
-
- err => {
- logger.error('Error in job handler %s.', job.handlerName, err)
- return this.onJobError(jobHandler, job, err)
- }
- )
- .then(() => callback(null))
- .catch(err => {
- this.cannotSaveJobError(err)
- return callback(err)
- })
+ await job.save()
+
+ try {
+ const result: T = await jobHandler.process(job.handlerInputData, job.id)
+ await this.onJobSuccess(jobHandler, job, result)
+ } catch (err) {
+ logger.error('Error in job handler %s.', job.handlerName, err)
+
+ try {
+ await this.onJobError(jobHandler, job, err)
+ } catch (innerErr) {
+ this.cannotSaveJobError(innerErr)
+ return callback(innerErr)
+ }
+ }
+
+ return callback(null)
}
- private onJobError (jobHandler: JobHandler<any>, job: JobInstance, err: Error) {
+ private async onJobError (jobHandler: JobHandler<P, T>, job: JobModel, err: Error) {
job.state = JOB_STATES.ERROR
- return job.save()
- .then(() => jobHandler.onError(err, job.id))
- .catch(err => this.cannotSaveJobError(err))
+ try {
+ await job.save()
+ if (jobHandler) await jobHandler.onError(err, job.id)
+ } catch (err) {
+ this.cannotSaveJobError(err)
+ }
}
- private onJobSuccess (jobHandler: JobHandler<any>, job: JobInstance, jobResult: any) {
+ private async onJobSuccess (jobHandler: JobHandler<P, T>, job: JobModel, jobResult: T) {
job.state = JOB_STATES.SUCCESS
- return job.save()
- .then(() => jobHandler.onSuccess(job.id, jobResult))
- .catch(err => this.cannotSaveJobError(err))
+ try {
+ await job.save()
+ await jobHandler.onSuccess(job.id, jobResult, this)
+ } catch (err) {
+ this.cannotSaveJobError(err)
+ }
}
private cannotSaveJobError (err: Error) {
projects / github / Chocobozzz / PeerTube.git / blobdiff