]>
Commit | Line | Data |
---|---|---|
1 | import Bull, { Job, JobOptions, Queue } from 'bull' | |
2 | import { jobStates } from '@server/helpers/custom-validators/jobs' | |
3 | import { CONFIG } from '@server/initializers/config' | |
4 | import { processVideoRedundancy } from '@server/lib/job-queue/handlers/video-redundancy' | |
5 | import { | |
6 | ActivitypubFollowPayload, | |
7 | ActivitypubHttpBroadcastPayload, | |
8 | ActivitypubHttpFetcherPayload, | |
9 | ActivitypubHttpUnicastPayload, | |
10 | ActorKeysPayload, | |
11 | DeleteResumableUploadMetaFilePayload, | |
12 | EmailPayload, | |
13 | JobState, | |
14 | JobType, | |
15 | ManageVideoTorrentPayload, | |
16 | MoveObjectStoragePayload, | |
17 | RefreshPayload, | |
18 | VideoFileImportPayload, | |
19 | VideoImportPayload, | |
20 | VideoLiveEndingPayload, | |
21 | VideoRedundancyPayload, | |
22 | VideoStudioEditionPayload, | |
23 | VideoTranscodingPayload | |
24 | } from '../../../shared/models' | |
25 | import { logger } from '../../helpers/logger' | |
26 | import { JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants' | |
27 | import { processActivityPubCleaner } from './handlers/activitypub-cleaner' | |
28 | import { processActivityPubFollow } from './handlers/activitypub-follow' | |
29 | import { processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' | |
30 | import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | |
31 | import { processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' | |
32 | import { refreshAPObject } from './handlers/activitypub-refresher' | |
33 | import { processActorKeys } from './handlers/actor-keys' | |
34 | import { processEmail } from './handlers/email' | |
35 | import { processManageVideoTorrent } from './handlers/manage-video-torrent' | |
36 | import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' | |
37 | import { processVideoFileImport } from './handlers/video-file-import' | |
38 | import { processVideoImport } from './handlers/video-import' | |
39 | import { processVideoLiveEnding } from './handlers/video-live-ending' | |
40 | import { processVideoStudioEdition } from './handlers/video-studio-edition' | |
41 | import { processVideoTranscoding } from './handlers/video-transcoding' | |
42 | import { processVideosViewsStats } from './handlers/video-views-stats' | |
43 | ||
44 | type CreateJobArgument = | |
45 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | |
46 | { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } | | |
47 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | | |
48 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | | |
49 | { type: 'activitypub-http-cleaner', payload: {} } | | |
50 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | | |
51 | { type: 'video-file-import', payload: VideoFileImportPayload } | | |
52 | { type: 'video-transcoding', payload: VideoTranscodingPayload } | | |
53 | { type: 'email', payload: EmailPayload } | | |
54 | { type: 'video-import', payload: VideoImportPayload } | | |
55 | { type: 'activitypub-refresher', payload: RefreshPayload } | | |
56 | { type: 'videos-views-stats', payload: {} } | | |
57 | { type: 'video-live-ending', payload: VideoLiveEndingPayload } | | |
58 | { type: 'actor-keys', payload: ActorKeysPayload } | | |
59 | { type: 'video-redundancy', payload: VideoRedundancyPayload } | | |
60 | { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } | | |
61 | { type: 'video-studio-edition', payload: VideoStudioEditionPayload } | | |
62 | { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } | | |
63 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | |
64 | ||
65 | export type CreateJobOptions = { | |
66 | delay?: number | |
67 | priority?: number | |
68 | } | |
69 | ||
70 | const handlers: { [id in JobType]: (job: Job) => Promise<any> } = { | |
71 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, | |
72 | 'activitypub-http-broadcast-parallel': processActivityPubHttpBroadcast, | |
73 | 'activitypub-http-unicast': processActivityPubHttpUnicast, | |
74 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | |
75 | 'activitypub-cleaner': processActivityPubCleaner, | |
76 | 'activitypub-follow': processActivityPubFollow, | |
77 | 'video-file-import': processVideoFileImport, | |
78 | 'video-transcoding': processVideoTranscoding, | |
79 | 'email': processEmail, | |
80 | 'video-import': processVideoImport, | |
81 | 'videos-views-stats': processVideosViewsStats, | |
82 | 'activitypub-refresher': refreshAPObject, | |
83 | 'video-live-ending': processVideoLiveEnding, | |
84 | 'actor-keys': processActorKeys, | |
85 | 'video-redundancy': processVideoRedundancy, | |
86 | 'move-to-object-storage': processMoveToObjectStorage, | |
87 | 'manage-video-torrent': processManageVideoTorrent, | |
88 | 'video-studio-edition': processVideoStudioEdition | |
89 | } | |
90 | ||
91 | const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = { | |
92 | 'move-to-object-storage': onMoveToObjectStorageFailure | |
93 | } | |
94 | ||
95 | const jobTypes: JobType[] = [ | |
96 | 'activitypub-follow', | |
97 | 'activitypub-http-broadcast', | |
98 | 'activitypub-http-broadcast-parallel', | |
99 | 'activitypub-http-fetcher', | |
100 | 'activitypub-http-unicast', | |
101 | 'activitypub-cleaner', | |
102 | 'email', | |
103 | 'video-transcoding', | |
104 | 'video-file-import', | |
105 | 'video-import', | |
106 | 'videos-views-stats', | |
107 | 'activitypub-refresher', | |
108 | 'video-redundancy', | |
109 | 'actor-keys', | |
110 | 'video-live-ending', | |
111 | 'move-to-object-storage', | |
112 | 'manage-video-torrent', | |
113 | 'video-studio-edition' | |
114 | ] | |
115 | ||
116 | const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ]) | |
117 | ||
118 | class JobQueue { | |
119 | ||
120 | private static instance: JobQueue | |
121 | ||
122 | private queues: { [id in JobType]?: Queue } = {} | |
123 | private initialized = false | |
124 | private jobRedisPrefix: string | |
125 | ||
126 | private constructor () { | |
127 | } | |
128 | ||
129 | init (produceOnly = false) { | |
130 | // Already initialized | |
131 | if (this.initialized === true) return | |
132 | this.initialized = true | |
133 | ||
134 | this.jobRedisPrefix = 'bull-' + WEBSERVER.HOST | |
135 | ||
136 | const queueOptions: Bull.QueueOptions = { | |
137 | prefix: this.jobRedisPrefix, | |
138 | redis: { | |
139 | password: CONFIG.REDIS.AUTH, | |
140 | db: CONFIG.REDIS.DB, | |
141 | host: CONFIG.REDIS.HOSTNAME, | |
142 | port: CONFIG.REDIS.PORT, | |
143 | path: CONFIG.REDIS.SOCKET | |
144 | }, | |
145 | settings: { | |
146 | maxStalledCount: 10 // transcoding could be long, so jobs can often be interrupted by restarts | |
147 | } | |
148 | } | |
149 | ||
150 | for (const handlerName of (Object.keys(handlers) as JobType[])) { | |
151 | const queue = new Bull(handlerName, queueOptions) | |
152 | ||
153 | if (produceOnly) { | |
154 | queue.pause(true) | |
155 | .catch(err => logger.error('Cannot pause queue %s in produced only job queue', handlerName, { err })) | |
156 | } | |
157 | ||
158 | const handler = handlers[handlerName] | |
159 | ||
160 | queue.process(this.getJobConcurrency(handlerName), handler) | |
161 | .catch(err => logger.error('Error in job queue processor %s.', handlerName, { err })) | |
162 | ||
163 | queue.on('failed', (job, err) => { | |
164 | const logLevel = silentFailure.has(handlerName) | |
165 | ? 'debug' | |
166 | : 'error' | |
167 | ||
168 | logger.log(logLevel, 'Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err }) | |
169 | ||
170 | if (errorHandlers[job.name]) { | |
171 | errorHandlers[job.name](job, err) | |
172 | .catch(err => logger.error('Cannot run error handler for job failure %d in queue %s.', job.id, handlerName, { err })) | |
173 | } | |
174 | }) | |
175 | ||
176 | queue.on('error', err => { | |
177 | logger.error('Error in job queue %s.', handlerName, { err }) | |
178 | }) | |
179 | ||
180 | this.queues[handlerName] = queue | |
181 | } | |
182 | ||
183 | this.addRepeatableJobs() | |
184 | } | |
185 | ||
186 | terminate () { | |
187 | for (const queueName of Object.keys(this.queues)) { | |
188 | const queue = this.queues[queueName] | |
189 | queue.close() | |
190 | } | |
191 | } | |
192 | ||
193 | async pause () { | |
194 | for (const handler of Object.keys(this.queues)) { | |
195 | await this.queues[handler].pause(true) | |
196 | } | |
197 | } | |
198 | ||
199 | async resume () { | |
200 | for (const handler of Object.keys(this.queues)) { | |
201 | await this.queues[handler].resume(true) | |
202 | } | |
203 | } | |
204 | ||
205 | createJob (obj: CreateJobArgument, options: CreateJobOptions = {}): void { | |
206 | this.createJobWithPromise(obj, options) | |
207 | .catch(err => logger.error('Cannot create job.', { err, obj })) | |
208 | } | |
209 | ||
210 | createJobWithPromise (obj: CreateJobArgument, options: CreateJobOptions = {}) { | |
211 | const queue: Queue = this.queues[obj.type] | |
212 | if (queue === undefined) { | |
213 | logger.error('Unknown queue %s: cannot create job.', obj.type) | |
214 | return | |
215 | } | |
216 | ||
217 | const jobArgs: JobOptions = { | |
218 | backoff: { delay: 60 * 1000, type: 'exponential' }, | |
219 | attempts: JOB_ATTEMPTS[obj.type], | |
220 | timeout: JOB_TTL[obj.type], | |
221 | priority: options.priority, | |
222 | delay: options.delay | |
223 | } | |
224 | ||
225 | return queue.add(obj.payload, jobArgs) | |
226 | } | |
227 | ||
228 | async listForApi (options: { | |
229 | state?: JobState | |
230 | start: number | |
231 | count: number | |
232 | asc?: boolean | |
233 | jobType: JobType | |
234 | }): Promise<Job[]> { | |
235 | const { state, start, count, asc, jobType } = options | |
236 | ||
237 | const states = state ? [ state ] : jobStates | |
238 | let results: Job[] = [] | |
239 | ||
240 | const filteredJobTypes = this.filterJobTypes(jobType) | |
241 | ||
242 | for (const jobType of filteredJobTypes) { | |
243 | const queue = this.queues[jobType] | |
244 | if (queue === undefined) { | |
245 | logger.error('Unknown queue %s to list jobs.', jobType) | |
246 | continue | |
247 | } | |
248 | ||
249 | const jobs = await queue.getJobs(states, 0, start + count, asc) | |
250 | results = results.concat(jobs) | |
251 | } | |
252 | ||
253 | results.sort((j1: any, j2: any) => { | |
254 | if (j1.timestamp < j2.timestamp) return -1 | |
255 | else if (j1.timestamp === j2.timestamp) return 0 | |
256 | ||
257 | return 1 | |
258 | }) | |
259 | ||
260 | if (asc === false) results.reverse() | |
261 | ||
262 | return results.slice(start, start + count) | |
263 | } | |
264 | ||
265 | async count (state: JobState, jobType?: JobType): Promise<number> { | |
266 | const states = state ? [ state ] : jobStates | |
267 | let total = 0 | |
268 | ||
269 | const filteredJobTypes = this.filterJobTypes(jobType) | |
270 | ||
271 | for (const type of filteredJobTypes) { | |
272 | const queue = this.queues[type] | |
273 | if (queue === undefined) { | |
274 | logger.error('Unknown queue %s to count jobs.', type) | |
275 | continue | |
276 | } | |
277 | ||
278 | const counts = await queue.getJobCounts() | |
279 | ||
280 | for (const s of states) { | |
281 | total += counts[s] | |
282 | } | |
283 | } | |
284 | ||
285 | return total | |
286 | } | |
287 | ||
288 | async getStats () { | |
289 | const promises = jobTypes.map(async t => ({ jobType: t, counts: await this.queues[t].getJobCounts() })) | |
290 | ||
291 | return Promise.all(promises) | |
292 | } | |
293 | ||
294 | async removeOldJobs () { | |
295 | for (const key of Object.keys(this.queues)) { | |
296 | const queue = this.queues[key] | |
297 | await queue.clean(JOB_COMPLETED_LIFETIME, 'completed') | |
298 | } | |
299 | } | |
300 | ||
301 | private addRepeatableJobs () { | |
302 | this.queues['videos-views-stats'].add({}, { | |
303 | repeat: REPEAT_JOBS['videos-views-stats'] | |
304 | }).catch(err => logger.error('Cannot add repeatable job.', { err })) | |
305 | ||
306 | if (CONFIG.FEDERATION.VIDEOS.CLEANUP_REMOTE_INTERACTIONS) { | |
307 | this.queues['activitypub-cleaner'].add({}, { | |
308 | repeat: REPEAT_JOBS['activitypub-cleaner'] | |
309 | }).catch(err => logger.error('Cannot add repeatable job.', { err })) | |
310 | } | |
311 | } | |
312 | ||
313 | private filterJobTypes (jobType?: JobType) { | |
314 | if (!jobType) return jobTypes | |
315 | ||
316 | return jobTypes.filter(t => t === jobType) | |
317 | } | |
318 | ||
319 | private getJobConcurrency (jobType: JobType) { | |
320 | if (jobType === 'video-transcoding') return CONFIG.TRANSCODING.CONCURRENCY | |
321 | if (jobType === 'video-import') return CONFIG.IMPORT.VIDEOS.CONCURRENCY | |
322 | ||
323 | return JOB_CONCURRENCY[jobType] | |
324 | } | |
325 | ||
326 | static get Instance () { | |
327 | return this.instance || (this.instance = new this()) | |
328 | } | |
329 | } | |
330 | ||
331 | // --------------------------------------------------------------------------- | |
332 | ||
333 | export { | |
334 | jobTypes, | |
335 | JobQueue | |
336 | } |