]>
Commit | Line | Data |
---|---|---|
41fb13c3 | 1 | import Bull, { Job, JobOptions, Queue } from 'bull' |
402145b8 | 2 | import { jobStates } from '@server/helpers/custom-validators/jobs' |
9129b769 | 3 | import { CONFIG } from '@server/initializers/config' |
402145b8 | 4 | import { processVideoRedundancy } from '@server/lib/job-queue/handlers/video-redundancy' |
8dc8a34e C |
5 | import { |
6 | ActivitypubFollowPayload, | |
7 | ActivitypubHttpBroadcastPayload, | |
e1c55031 C |
8 | ActivitypubHttpFetcherPayload, |
9 | ActivitypubHttpUnicastPayload, | |
8795d6f2 | 10 | ActorKeysPayload, |
276250f0 | 11 | DeleteResumableUploadMetaFilePayload, |
e1c55031 | 12 | EmailPayload, |
8dc8a34e | 13 | JobState, |
e1c55031 | 14 | JobType, |
f012319a | 15 | ManageVideoTorrentPayload, |
0305db28 | 16 | MoveObjectStoragePayload, |
e1c55031 C |
17 | RefreshPayload, |
18 | VideoFileImportPayload, | |
19 | VideoImportPayload, | |
a5cf76af | 20 | VideoLiveEndingPayload, |
e1c55031 | 21 | VideoRedundancyPayload, |
92e66e04 | 22 | VideoStudioEditionPayload, |
e1c55031 | 23 | VideoTranscodingPayload |
8dc8a34e | 24 | } from '../../../shared/models' |
94a5ff8a | 25 | import { logger } from '../../helpers/logger' |
74dc3bca | 26 | import { JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants' |
22df69fd | 27 | import { Hooks } from '../plugins/hooks' |
74d249bc | 28 | import { processActivityPubCleaner } from './handlers/activitypub-cleaner' |
402145b8 | 29 | import { processActivityPubFollow } from './handlers/activitypub-follow' |
8dc8a34e C |
30 | import { processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' |
31 | import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | |
32 | import { processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' | |
e1c55031 | 33 | import { refreshAPObject } from './handlers/activitypub-refresher' |
8795d6f2 | 34 | import { processActorKeys } from './handlers/actor-keys' |
402145b8 | 35 | import { processEmail } from './handlers/email' |
f012319a | 36 | import { processManageVideoTorrent } from './handlers/manage-video-torrent' |
32567717 | 37 | import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' |
e1c55031 | 38 | import { processVideoFileImport } from './handlers/video-file-import' |
402145b8 | 39 | import { processVideoImport } from './handlers/video-import' |
a5cf76af | 40 | import { processVideoLiveEnding } from './handlers/video-live-ending' |
92e66e04 | 41 | import { processVideoStudioEdition } from './handlers/video-studio-edition' |
402145b8 | 42 | import { processVideoTranscoding } from './handlers/video-transcoding' |
51353d9a | 43 | import { processVideosViewsStats } from './handlers/video-views-stats' |
94a5ff8a C |
44 | |
45 | type CreateJobArgument = | |
46 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | |
f27b7a75 | 47 | { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } | |
94a5ff8a C |
48 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | |
49 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | | |
74d249bc | 50 | { type: 'activitypub-http-cleaner', payload: {} } | |
5350fd8e | 51 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | |
28be8916 | 52 | { type: 'video-file-import', payload: VideoFileImportPayload } | |
a0327eed | 53 | { type: 'video-transcoding', payload: VideoTranscodingPayload } | |
fbad87b0 | 54 | { type: 'email', payload: EmailPayload } | |
6b616860 | 55 | { type: 'video-import', payload: VideoImportPayload } | |
04b8c3fb | 56 | { type: 'activitypub-refresher', payload: RefreshPayload } | |
51353d9a | 57 | { type: 'videos-views-stats', payload: {} } | |
a5cf76af | 58 | { type: 'video-live-ending', payload: VideoLiveEndingPayload } | |
8795d6f2 | 59 | { type: 'actor-keys', payload: ActorKeysPayload } | |
0305db28 | 60 | { type: 'video-redundancy', payload: VideoRedundancyPayload } | |
276250f0 | 61 | { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } | |
92e66e04 | 62 | { type: 'video-studio-edition', payload: VideoStudioEditionPayload } | |
f012319a | 63 | { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } | |
0305db28 | 64 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
94a5ff8a | 65 | |
0305db28 | 66 | export type CreateJobOptions = { |
a5cf76af | 67 | delay?: number |
77d7e851 | 68 | priority?: number |
a5cf76af C |
69 | } |
70 | ||
41fb13c3 | 71 | const handlers: { [id in JobType]: (job: Job) => Promise<any> } = { |
94a5ff8a | 72 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, |
f27b7a75 | 73 | 'activitypub-http-broadcast-parallel': processActivityPubHttpBroadcast, |
94a5ff8a C |
74 | 'activitypub-http-unicast': processActivityPubHttpUnicast, |
75 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | |
74d249bc | 76 | 'activitypub-cleaner': processActivityPubCleaner, |
5350fd8e | 77 | 'activitypub-follow': processActivityPubFollow, |
28be8916 | 78 | 'video-file-import': processVideoFileImport, |
a0327eed | 79 | 'video-transcoding': processVideoTranscoding, |
fbad87b0 | 80 | 'email': processEmail, |
6b616860 | 81 | 'video-import': processVideoImport, |
51353d9a | 82 | 'videos-views-stats': processVideosViewsStats, |
b764380a | 83 | 'activitypub-refresher': refreshAPObject, |
a5cf76af | 84 | 'video-live-ending': processVideoLiveEnding, |
8795d6f2 | 85 | 'actor-keys': processActorKeys, |
0305db28 | 86 | 'video-redundancy': processVideoRedundancy, |
c729caf6 | 87 | 'move-to-object-storage': processMoveToObjectStorage, |
f012319a | 88 | 'manage-video-torrent': processManageVideoTorrent, |
92e66e04 | 89 | 'video-studio-edition': processVideoStudioEdition |
94a5ff8a C |
90 | } |
91 | ||
32567717 C |
92 | const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = { |
93 | 'move-to-object-storage': onMoveToObjectStorageFailure | |
94 | } | |
95 | ||
94831479 C |
96 | const jobTypes: JobType[] = [ |
97 | 'activitypub-follow', | |
71e3dfda | 98 | 'activitypub-http-broadcast', |
f27b7a75 | 99 | 'activitypub-http-broadcast-parallel', |
71e3dfda | 100 | 'activitypub-http-fetcher', |
94831479 | 101 | 'activitypub-http-unicast', |
74d249bc | 102 | 'activitypub-cleaner', |
94831479 | 103 | 'email', |
a0327eed | 104 | 'video-transcoding', |
fbad87b0 | 105 | 'video-file-import', |
6b616860 | 106 | 'video-import', |
51353d9a | 107 | 'videos-views-stats', |
b764380a | 108 | 'activitypub-refresher', |
a5cf76af | 109 | 'video-redundancy', |
8795d6f2 | 110 | 'actor-keys', |
0305db28 | 111 | 'video-live-ending', |
c729caf6 | 112 | 'move-to-object-storage', |
f012319a | 113 | 'manage-video-torrent', |
92e66e04 | 114 | 'video-studio-edition' |
71e3dfda C |
115 | ] |
116 | ||
941d28cc C |
117 | const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ]) |
118 | ||
94a5ff8a C |
119 | class JobQueue { |
120 | ||
121 | private static instance: JobQueue | |
122 | ||
41fb13c3 | 123 | private queues: { [id in JobType]?: Queue } = {} |
94a5ff8a | 124 | private initialized = false |
2c29ad4f | 125 | private jobRedisPrefix: string |
94a5ff8a | 126 | |
a1587156 C |
127 | private constructor () { |
128 | } | |
94a5ff8a | 129 | |
e1ab52d7 | 130 | init (produceOnly = false) { |
94a5ff8a C |
131 | // Already initialized |
132 | if (this.initialized === true) return | |
133 | this.initialized = true | |
134 | ||
6dd9de95 | 135 | this.jobRedisPrefix = 'bull-' + WEBSERVER.HOST |
ff4d2c73 C |
136 | |
137 | const queueOptions: Bull.QueueOptions = { | |
2c29ad4f | 138 | prefix: this.jobRedisPrefix, |
ff4d2c73 C |
139 | redis: { |
140 | password: CONFIG.REDIS.AUTH, | |
141 | db: CONFIG.REDIS.DB, | |
142 | host: CONFIG.REDIS.HOSTNAME, | |
143 | port: CONFIG.REDIS.PORT, | |
144 | path: CONFIG.REDIS.SOCKET | |
145 | }, | |
4a9e71c2 C |
146 | settings: { |
147 | maxStalledCount: 10 // transcoding could be long, so jobs can often be interrupted by restarts | |
148 | } | |
94831479 | 149 | } |
ecb4e35f | 150 | |
9129b769 | 151 | for (const handlerName of (Object.keys(handlers) as JobType[])) { |
94831479 | 152 | const queue = new Bull(handlerName, queueOptions) |
e1ab52d7 | 153 | |
154 | if (produceOnly) { | |
155 | queue.pause(true) | |
156 | .catch(err => logger.error('Cannot pause queue %s in produced only job queue', handlerName, { err })) | |
157 | } | |
158 | ||
94831479 | 159 | const handler = handlers[handlerName] |
94a5ff8a | 160 | |
22df69fd C |
161 | queue.process(this.getJobConcurrency(handlerName), async (jobArg: Job<any>) => { |
162 | const job = await Hooks.wrapObject(jobArg, 'filter:job-queue.process.params', { type: handlerName }) | |
163 | ||
164 | return Hooks.wrapPromiseFun(handler, job, 'filter:job-queue.process.result') | |
165 | }).catch(err => logger.error('Error in job queue processor %s.', handlerName, { err })) | |
d7f83948 C |
166 | |
167 | queue.on('failed', (job, err) => { | |
941d28cc C |
168 | const logLevel = silentFailure.has(handlerName) |
169 | ? 'debug' | |
170 | : 'error' | |
171 | ||
172 | logger.log(logLevel, 'Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err }) | |
32567717 C |
173 | |
174 | if (errorHandlers[job.name]) { | |
175 | errorHandlers[job.name](job, err) | |
176 | .catch(err => logger.error('Cannot run error handler for job failure %d in queue %s.', job.id, handlerName, { err })) | |
177 | } | |
d7f83948 | 178 | }) |
3df45638 | 179 | |
94831479 C |
180 | queue.on('error', err => { |
181 | logger.error('Error in job queue %s.', handlerName, { err }) | |
94a5ff8a | 182 | }) |
94831479 C |
183 | |
184 | this.queues[handlerName] = queue | |
94a5ff8a | 185 | } |
6b616860 C |
186 | |
187 | this.addRepeatableJobs() | |
94a5ff8a C |
188 | } |
189 | ||
14f2b3ad C |
190 | terminate () { |
191 | for (const queueName of Object.keys(this.queues)) { | |
192 | const queue = this.queues[queueName] | |
193 | queue.close() | |
194 | } | |
195 | } | |
196 | ||
419b520c C |
197 | async pause () { |
198 | for (const handler of Object.keys(this.queues)) { | |
199 | await this.queues[handler].pause(true) | |
200 | } | |
201 | } | |
202 | ||
203 | async resume () { | |
204 | for (const handler of Object.keys(this.queues)) { | |
205 | await this.queues[handler].resume(true) | |
206 | } | |
207 | } | |
208 | ||
a5cf76af C |
209 | createJob (obj: CreateJobArgument, options: CreateJobOptions = {}): void { |
210 | this.createJobWithPromise(obj, options) | |
e1c55031 | 211 | .catch(err => logger.error('Cannot create job.', { err, obj })) |
a1587156 C |
212 | } |
213 | ||
a5cf76af | 214 | createJobWithPromise (obj: CreateJobArgument, options: CreateJobOptions = {}) { |
f012319a | 215 | const queue: Queue = this.queues[obj.type] |
94831479 C |
216 | if (queue === undefined) { |
217 | logger.error('Unknown queue %s: cannot create job.', obj.type) | |
a1587156 | 218 | return |
94831479 | 219 | } |
94a5ff8a | 220 | |
41fb13c3 | 221 | const jobArgs: JobOptions = { |
94831479 | 222 | backoff: { delay: 60 * 1000, type: 'exponential' }, |
2b86fe72 | 223 | attempts: JOB_ATTEMPTS[obj.type], |
a5cf76af | 224 | timeout: JOB_TTL[obj.type], |
77d7e851 | 225 | priority: options.priority, |
a5cf76af | 226 | delay: options.delay |
94831479 | 227 | } |
71e3dfda | 228 | |
94831479 | 229 | return queue.add(obj.payload, jobArgs) |
94a5ff8a C |
230 | } |
231 | ||
1061c73f | 232 | async listForApi (options: { |
402145b8 | 233 | state?: JobState |
a1587156 C |
234 | start: number |
235 | count: number | |
236 | asc?: boolean | |
1061c73f | 237 | jobType: JobType |
41fb13c3 | 238 | }): Promise<Job[]> { |
402145b8 C |
239 | const { state, start, count, asc, jobType } = options |
240 | ||
241 | const states = state ? [ state ] : jobStates | |
41fb13c3 | 242 | let results: Job[] = [] |
94a5ff8a | 243 | |
1061c73f C |
244 | const filteredJobTypes = this.filterJobTypes(jobType) |
245 | ||
1061c73f | 246 | for (const jobType of filteredJobTypes) { |
a1587156 | 247 | const queue = this.queues[jobType] |
94831479 C |
248 | if (queue === undefined) { |
249 | logger.error('Unknown queue %s to list jobs.', jobType) | |
250 | continue | |
251 | } | |
2c29ad4f | 252 | |
402145b8 | 253 | const jobs = await queue.getJobs(states, 0, start + count, asc) |
94831479 C |
254 | results = results.concat(jobs) |
255 | } | |
94a5ff8a | 256 | |
94831479 C |
257 | results.sort((j1: any, j2: any) => { |
258 | if (j1.timestamp < j2.timestamp) return -1 | |
259 | else if (j1.timestamp === j2.timestamp) return 0 | |
94a5ff8a | 260 | |
94831479 | 261 | return 1 |
94a5ff8a | 262 | }) |
94a5ff8a | 263 | |
94831479 | 264 | if (asc === false) results.reverse() |
94a5ff8a | 265 | |
94831479 | 266 | return results.slice(start, start + count) |
94a5ff8a C |
267 | } |
268 | ||
402145b8 C |
269 | async count (state: JobState, jobType?: JobType): Promise<number> { |
270 | const states = state ? [ state ] : jobStates | |
94831479 | 271 | let total = 0 |
3df45638 | 272 | |
1061c73f C |
273 | const filteredJobTypes = this.filterJobTypes(jobType) |
274 | ||
275 | for (const type of filteredJobTypes) { | |
a1587156 | 276 | const queue = this.queues[type] |
94831479 C |
277 | if (queue === undefined) { |
278 | logger.error('Unknown queue %s to count jobs.', type) | |
279 | continue | |
280 | } | |
3df45638 | 281 | |
94831479 | 282 | const counts = await queue.getJobCounts() |
3df45638 | 283 | |
040d6896 RK |
284 | for (const s of states) { |
285 | total += counts[s] | |
286 | } | |
94831479 | 287 | } |
3df45638 | 288 | |
94831479 | 289 | return total |
3df45638 C |
290 | } |
291 | ||
630d0a1b C |
292 | async getStats () { |
293 | const promises = jobTypes.map(async t => ({ jobType: t, counts: await this.queues[t].getJobCounts() })) | |
294 | ||
295 | return Promise.all(promises) | |
296 | } | |
297 | ||
2f5c6b2f | 298 | async removeOldJobs () { |
94831479 C |
299 | for (const key of Object.keys(this.queues)) { |
300 | const queue = this.queues[key] | |
2f5c6b2f | 301 | await queue.clean(JOB_COMPLETED_LIFETIME, 'completed') |
94831479 | 302 | } |
2c29ad4f C |
303 | } |
304 | ||
6b616860 | 305 | private addRepeatableJobs () { |
51353d9a C |
306 | this.queues['videos-views-stats'].add({}, { |
307 | repeat: REPEAT_JOBS['videos-views-stats'] | |
a1587156 | 308 | }).catch(err => logger.error('Cannot add repeatable job.', { err })) |
74d249bc C |
309 | |
310 | if (CONFIG.FEDERATION.VIDEOS.CLEANUP_REMOTE_INTERACTIONS) { | |
311 | this.queues['activitypub-cleaner'].add({}, { | |
312 | repeat: REPEAT_JOBS['activitypub-cleaner'] | |
313 | }).catch(err => logger.error('Cannot add repeatable job.', { err })) | |
314 | } | |
6b616860 C |
315 | } |
316 | ||
1061c73f C |
317 | private filterJobTypes (jobType?: JobType) { |
318 | if (!jobType) return jobTypes | |
319 | ||
320 | return jobTypes.filter(t => t === jobType) | |
321 | } | |
322 | ||
9129b769 C |
323 | private getJobConcurrency (jobType: JobType) { |
324 | if (jobType === 'video-transcoding') return CONFIG.TRANSCODING.CONCURRENCY | |
325 | if (jobType === 'video-import') return CONFIG.IMPORT.VIDEOS.CONCURRENCY | |
326 | ||
327 | return JOB_CONCURRENCY[jobType] | |
328 | } | |
329 | ||
94a5ff8a C |
330 | static get Instance () { |
331 | return this.instance || (this.instance = new this()) | |
332 | } | |
333 | } | |
334 | ||
335 | // --------------------------------------------------------------------------- | |
336 | ||
337 | export { | |
1061c73f | 338 | jobTypes, |
94a5ff8a C |
339 | JobQueue |
340 | } |