]>
Commit | Line | Data |
---|---|---|
94831479 | 1 | import * as Bull from 'bull' |
402145b8 C |
2 | import { jobStates } from '@server/helpers/custom-validators/jobs' |
3 | import { processVideoRedundancy } from '@server/lib/job-queue/handlers/video-redundancy' | |
8dc8a34e C |
4 | import { |
5 | ActivitypubFollowPayload, | |
6 | ActivitypubHttpBroadcastPayload, | |
e1c55031 C |
7 | ActivitypubHttpFetcherPayload, |
8 | ActivitypubHttpUnicastPayload, | |
9 | EmailPayload, | |
8dc8a34e | 10 | JobState, |
e1c55031 C |
11 | JobType, |
12 | RefreshPayload, | |
13 | VideoFileImportPayload, | |
14 | VideoImportPayload, | |
a5cf76af | 15 | VideoLiveEndingPayload, |
e1c55031 C |
16 | VideoRedundancyPayload, |
17 | VideoTranscodingPayload | |
8dc8a34e | 18 | } from '../../../shared/models' |
94a5ff8a | 19 | import { logger } from '../../helpers/logger' |
74dc3bca | 20 | import { JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants' |
402145b8 C |
21 | import { Redis } from '../redis' |
22 | import { processActivityPubFollow } from './handlers/activitypub-follow' | |
8dc8a34e C |
23 | import { processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' |
24 | import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | |
25 | import { processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' | |
e1c55031 | 26 | import { refreshAPObject } from './handlers/activitypub-refresher' |
402145b8 | 27 | import { processEmail } from './handlers/email' |
e1c55031 | 28 | import { processVideoFileImport } from './handlers/video-file-import' |
402145b8 | 29 | import { processVideoImport } from './handlers/video-import' |
a5cf76af | 30 | import { processVideoLiveEnding } from './handlers/video-live-ending' |
402145b8 C |
31 | import { processVideoTranscoding } from './handlers/video-transcoding' |
32 | import { processVideosViews } from './handlers/video-views' | |
94a5ff8a C |
33 | |
34 | type CreateJobArgument = | |
35 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | |
36 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | | |
37 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | | |
5350fd8e | 38 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | |
28be8916 | 39 | { type: 'video-file-import', payload: VideoFileImportPayload } | |
a0327eed | 40 | { type: 'video-transcoding', payload: VideoTranscodingPayload } | |
fbad87b0 | 41 | { type: 'email', payload: EmailPayload } | |
6b616860 | 42 | { type: 'video-import', payload: VideoImportPayload } | |
04b8c3fb | 43 | { type: 'activitypub-refresher', payload: RefreshPayload } | |
b764380a | 44 | { type: 'videos-views', payload: {} } | |
a5cf76af | 45 | { type: 'video-live-ending', payload: VideoLiveEndingPayload } | |
b764380a | 46 | { type: 'video-redundancy', payload: VideoRedundancyPayload } |
94a5ff8a | 47 | |
a5cf76af C |
48 | type CreateJobOptions = { |
49 | delay?: number | |
77d7e851 | 50 | priority?: number |
a5cf76af C |
51 | } |
52 | ||
a1587156 | 53 | const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = { |
94a5ff8a C |
54 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, |
55 | 'activitypub-http-unicast': processActivityPubHttpUnicast, | |
56 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | |
5350fd8e | 57 | 'activitypub-follow': processActivityPubFollow, |
28be8916 | 58 | 'video-file-import': processVideoFileImport, |
a0327eed | 59 | 'video-transcoding': processVideoTranscoding, |
fbad87b0 | 60 | 'email': processEmail, |
6b616860 | 61 | 'video-import': processVideoImport, |
04b8c3fb | 62 | 'videos-views': processVideosViews, |
b764380a | 63 | 'activitypub-refresher': refreshAPObject, |
a5cf76af | 64 | 'video-live-ending': processVideoLiveEnding, |
b764380a | 65 | 'video-redundancy': processVideoRedundancy |
94a5ff8a C |
66 | } |
67 | ||
94831479 C |
68 | const jobTypes: JobType[] = [ |
69 | 'activitypub-follow', | |
71e3dfda | 70 | 'activitypub-http-broadcast', |
71e3dfda | 71 | 'activitypub-http-fetcher', |
94831479 C |
72 | 'activitypub-http-unicast', |
73 | 'email', | |
a0327eed | 74 | 'video-transcoding', |
fbad87b0 | 75 | 'video-file-import', |
6b616860 | 76 | 'video-import', |
04b8c3fb | 77 | 'videos-views', |
b764380a | 78 | 'activitypub-refresher', |
a5cf76af C |
79 | 'video-redundancy', |
80 | 'video-live-ending' | |
71e3dfda C |
81 | ] |
82 | ||
94a5ff8a C |
83 | class JobQueue { |
84 | ||
85 | private static instance: JobQueue | |
86 | ||
a1587156 | 87 | private queues: { [id in JobType]?: Bull.Queue } = {} |
94a5ff8a | 88 | private initialized = false |
2c29ad4f | 89 | private jobRedisPrefix: string |
94a5ff8a | 90 | |
a1587156 C |
91 | private constructor () { |
92 | } | |
94a5ff8a | 93 | |
a1587156 | 94 | init () { |
94a5ff8a C |
95 | // Already initialized |
96 | if (this.initialized === true) return | |
97 | this.initialized = true | |
98 | ||
6dd9de95 | 99 | this.jobRedisPrefix = 'bull-' + WEBSERVER.HOST |
94831479 | 100 | const queueOptions = { |
2c29ad4f | 101 | prefix: this.jobRedisPrefix, |
47f6409b | 102 | redis: Redis.getRedisClientOptions(), |
4a9e71c2 C |
103 | settings: { |
104 | maxStalledCount: 10 // transcoding could be long, so jobs can often be interrupted by restarts | |
105 | } | |
94831479 | 106 | } |
ecb4e35f | 107 | |
94831479 C |
108 | for (const handlerName of Object.keys(handlers)) { |
109 | const queue = new Bull(handlerName, queueOptions) | |
110 | const handler = handlers[handlerName] | |
94a5ff8a | 111 | |
94831479 | 112 | queue.process(JOB_CONCURRENCY[handlerName], handler) |
2b86fe72 | 113 | .catch(err => logger.error('Error in job queue processor %s.', handlerName, { err })) |
d7f83948 C |
114 | |
115 | queue.on('failed', (job, err) => { | |
116 | logger.error('Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err }) | |
117 | }) | |
3df45638 | 118 | |
94831479 C |
119 | queue.on('error', err => { |
120 | logger.error('Error in job queue %s.', handlerName, { err }) | |
94a5ff8a | 121 | }) |
94831479 C |
122 | |
123 | this.queues[handlerName] = queue | |
94a5ff8a | 124 | } |
6b616860 C |
125 | |
126 | this.addRepeatableJobs() | |
94a5ff8a C |
127 | } |
128 | ||
14f2b3ad C |
129 | terminate () { |
130 | for (const queueName of Object.keys(this.queues)) { | |
131 | const queue = this.queues[queueName] | |
132 | queue.close() | |
133 | } | |
134 | } | |
135 | ||
a5cf76af C |
136 | createJob (obj: CreateJobArgument, options: CreateJobOptions = {}): void { |
137 | this.createJobWithPromise(obj, options) | |
e1c55031 | 138 | .catch(err => logger.error('Cannot create job.', { err, obj })) |
a1587156 C |
139 | } |
140 | ||
a5cf76af | 141 | createJobWithPromise (obj: CreateJobArgument, options: CreateJobOptions = {}) { |
94831479 C |
142 | const queue = this.queues[obj.type] |
143 | if (queue === undefined) { | |
144 | logger.error('Unknown queue %s: cannot create job.', obj.type) | |
a1587156 | 145 | return |
94831479 | 146 | } |
94a5ff8a | 147 | |
94831479 C |
148 | const jobArgs: Bull.JobOptions = { |
149 | backoff: { delay: 60 * 1000, type: 'exponential' }, | |
2b86fe72 | 150 | attempts: JOB_ATTEMPTS[obj.type], |
a5cf76af | 151 | timeout: JOB_TTL[obj.type], |
77d7e851 | 152 | priority: options.priority, |
a5cf76af | 153 | delay: options.delay |
94831479 | 154 | } |
71e3dfda | 155 | |
94831479 | 156 | return queue.add(obj.payload, jobArgs) |
94a5ff8a C |
157 | } |
158 | ||
1061c73f | 159 | async listForApi (options: { |
402145b8 | 160 | state?: JobState |
a1587156 C |
161 | start: number |
162 | count: number | |
163 | asc?: boolean | |
1061c73f C |
164 | jobType: JobType |
165 | }): Promise<Bull.Job[]> { | |
402145b8 C |
166 | const { state, start, count, asc, jobType } = options |
167 | ||
168 | const states = state ? [ state ] : jobStates | |
94831479 | 169 | let results: Bull.Job[] = [] |
94a5ff8a | 170 | |
1061c73f C |
171 | const filteredJobTypes = this.filterJobTypes(jobType) |
172 | ||
1061c73f | 173 | for (const jobType of filteredJobTypes) { |
a1587156 | 174 | const queue = this.queues[jobType] |
94831479 C |
175 | if (queue === undefined) { |
176 | logger.error('Unknown queue %s to list jobs.', jobType) | |
177 | continue | |
178 | } | |
2c29ad4f | 179 | |
402145b8 | 180 | const jobs = await queue.getJobs(states, 0, start + count, asc) |
94831479 C |
181 | results = results.concat(jobs) |
182 | } | |
94a5ff8a | 183 | |
94831479 C |
184 | results.sort((j1: any, j2: any) => { |
185 | if (j1.timestamp < j2.timestamp) return -1 | |
186 | else if (j1.timestamp === j2.timestamp) return 0 | |
94a5ff8a | 187 | |
94831479 | 188 | return 1 |
94a5ff8a | 189 | }) |
94a5ff8a | 190 | |
94831479 | 191 | if (asc === false) results.reverse() |
94a5ff8a | 192 | |
94831479 | 193 | return results.slice(start, start + count) |
94a5ff8a C |
194 | } |
195 | ||
402145b8 C |
196 | async count (state: JobState, jobType?: JobType): Promise<number> { |
197 | const states = state ? [ state ] : jobStates | |
94831479 | 198 | let total = 0 |
3df45638 | 199 | |
1061c73f C |
200 | const filteredJobTypes = this.filterJobTypes(jobType) |
201 | ||
202 | for (const type of filteredJobTypes) { | |
a1587156 | 203 | const queue = this.queues[type] |
94831479 C |
204 | if (queue === undefined) { |
205 | logger.error('Unknown queue %s to count jobs.', type) | |
206 | continue | |
207 | } | |
3df45638 | 208 | |
94831479 | 209 | const counts = await queue.getJobCounts() |
3df45638 | 210 | |
040d6896 RK |
211 | for (const s of states) { |
212 | total += counts[s] | |
213 | } | |
94831479 | 214 | } |
3df45638 | 215 | |
94831479 | 216 | return total |
3df45638 C |
217 | } |
218 | ||
2f5c6b2f | 219 | async removeOldJobs () { |
94831479 C |
220 | for (const key of Object.keys(this.queues)) { |
221 | const queue = this.queues[key] | |
2f5c6b2f | 222 | await queue.clean(JOB_COMPLETED_LIFETIME, 'completed') |
94831479 | 223 | } |
2c29ad4f C |
224 | } |
225 | ||
6b616860 C |
226 | private addRepeatableJobs () { |
227 | this.queues['videos-views'].add({}, { | |
228 | repeat: REPEAT_JOBS['videos-views'] | |
a1587156 | 229 | }).catch(err => logger.error('Cannot add repeatable job.', { err })) |
6b616860 C |
230 | } |
231 | ||
1061c73f C |
232 | private filterJobTypes (jobType?: JobType) { |
233 | if (!jobType) return jobTypes | |
234 | ||
235 | return jobTypes.filter(t => t === jobType) | |
236 | } | |
237 | ||
94a5ff8a C |
238 | static get Instance () { |
239 | return this.instance || (this.instance = new this()) | |
240 | } | |
241 | } | |
242 | ||
243 | // --------------------------------------------------------------------------- | |
244 | ||
245 | export { | |
1061c73f | 246 | jobTypes, |
94a5ff8a C |
247 | JobQueue |
248 | } |