]>
Commit | Line | Data |
---|---|---|
94831479 | 1 | import * as Bull from 'bull' |
628d28e8 | 2 | import { JobState, JobType } from '../../../shared/models' |
94a5ff8a | 3 | import { logger } from '../../helpers/logger' |
19f7b248 | 4 | import { Redis } from '../redis' |
6b616860 | 5 | import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS } from '../../initializers' |
94a5ff8a C |
6 | import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' |
7 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | |
8 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' | |
ecb4e35f | 9 | import { EmailPayload, processEmail } from './handlers/email' |
94831479 | 10 | import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' |
5350fd8e | 11 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' |
fbad87b0 | 12 | import { processVideoImport, VideoImportPayload } from './handlers/video-import' |
030177d2 | 13 | import { processVideosViews } from './handlers/video-views' |
04b8c3fb | 14 | import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' |
94a5ff8a C |
15 | |
16 | type CreateJobArgument = | |
17 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | |
18 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | | |
19 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | | |
5350fd8e | 20 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | |
28be8916 | 21 | { type: 'video-file-import', payload: VideoFileImportPayload } | |
ecb4e35f | 22 | { type: 'video-file', payload: VideoFilePayload } | |
fbad87b0 | 23 | { type: 'email', payload: EmailPayload } | |
6b616860 | 24 | { type: 'video-import', payload: VideoImportPayload } | |
04b8c3fb | 25 | { type: 'activitypub-refresher', payload: RefreshPayload } | |
6b616860 | 26 | { type: 'videos-views', payload: {} } |
94a5ff8a | 27 | |
94831479 | 28 | const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { |
94a5ff8a C |
29 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, |
30 | 'activitypub-http-unicast': processActivityPubHttpUnicast, | |
31 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | |
5350fd8e | 32 | 'activitypub-follow': processActivityPubFollow, |
28be8916 | 33 | 'video-file-import': processVideoFileImport, |
ecb4e35f | 34 | 'video-file': processVideoFile, |
fbad87b0 | 35 | 'email': processEmail, |
6b616860 | 36 | 'video-import': processVideoImport, |
04b8c3fb C |
37 | 'videos-views': processVideosViews, |
38 | 'activitypub-refresher': refreshAPObject | |
94a5ff8a C |
39 | } |
40 | ||
94831479 C |
41 | const jobTypes: JobType[] = [ |
42 | 'activitypub-follow', | |
71e3dfda | 43 | 'activitypub-http-broadcast', |
71e3dfda | 44 | 'activitypub-http-fetcher', |
94831479 C |
45 | 'activitypub-http-unicast', |
46 | 'email', | |
47 | 'video-file', | |
fbad87b0 | 48 | 'video-file-import', |
6b616860 | 49 | 'video-import', |
04b8c3fb C |
50 | 'videos-views', |
51 | 'activitypub-refresher' | |
71e3dfda C |
52 | ] |
53 | ||
94a5ff8a C |
54 | class JobQueue { |
55 | ||
56 | private static instance: JobQueue | |
57 | ||
94831479 | 58 | private queues: { [ id in JobType ]?: Bull.Queue } = {} |
94a5ff8a | 59 | private initialized = false |
2c29ad4f | 60 | private jobRedisPrefix: string |
94a5ff8a C |
61 | |
62 | private constructor () {} | |
63 | ||
3df45638 | 64 | async init () { |
94a5ff8a C |
65 | // Already initialized |
66 | if (this.initialized === true) return | |
67 | this.initialized = true | |
68 | ||
94831479 C |
69 | this.jobRedisPrefix = 'bull-' + CONFIG.WEBSERVER.HOST |
70 | const queueOptions = { | |
2c29ad4f | 71 | prefix: this.jobRedisPrefix, |
4a9e71c2 C |
72 | redis: Redis.getRedisClient(), |
73 | settings: { | |
74 | maxStalledCount: 10 // transcoding could be long, so jobs can often be interrupted by restarts | |
75 | } | |
94831479 | 76 | } |
ecb4e35f | 77 | |
94831479 C |
78 | for (const handlerName of Object.keys(handlers)) { |
79 | const queue = new Bull(handlerName, queueOptions) | |
80 | const handler = handlers[handlerName] | |
94a5ff8a | 81 | |
94831479 | 82 | queue.process(JOB_CONCURRENCY[handlerName], handler) |
2b86fe72 | 83 | .catch(err => logger.error('Error in job queue processor %s.', handlerName, { err })) |
d7f83948 C |
84 | |
85 | queue.on('failed', (job, err) => { | |
86 | logger.error('Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err }) | |
87 | }) | |
3df45638 | 88 | |
94831479 C |
89 | queue.on('error', err => { |
90 | logger.error('Error in job queue %s.', handlerName, { err }) | |
94a5ff8a | 91 | }) |
94831479 C |
92 | |
93 | this.queues[handlerName] = queue | |
94a5ff8a | 94 | } |
6b616860 C |
95 | |
96 | this.addRepeatableJobs() | |
94a5ff8a C |
97 | } |
98 | ||
14f2b3ad C |
99 | terminate () { |
100 | for (const queueName of Object.keys(this.queues)) { | |
101 | const queue = this.queues[queueName] | |
102 | queue.close() | |
103 | } | |
104 | } | |
105 | ||
94831479 C |
106 | createJob (obj: CreateJobArgument) { |
107 | const queue = this.queues[obj.type] | |
108 | if (queue === undefined) { | |
109 | logger.error('Unknown queue %s: cannot create job.', obj.type) | |
c1e791ba | 110 | throw Error('Unknown queue, cannot create job') |
94831479 | 111 | } |
94a5ff8a | 112 | |
94831479 C |
113 | const jobArgs: Bull.JobOptions = { |
114 | backoff: { delay: 60 * 1000, type: 'exponential' }, | |
2b86fe72 C |
115 | attempts: JOB_ATTEMPTS[obj.type], |
116 | timeout: JOB_TTL[obj.type] | |
94831479 | 117 | } |
71e3dfda | 118 | |
94831479 | 119 | return queue.add(obj.payload, jobArgs) |
94a5ff8a C |
120 | } |
121 | ||
94831479 C |
122 | async listForApi (state: JobState, start: number, count: number, asc?: boolean): Promise<Bull.Job[]> { |
123 | let results: Bull.Job[] = [] | |
94a5ff8a | 124 | |
94831479 C |
125 | // TODO: optimize |
126 | for (const jobType of jobTypes) { | |
127 | const queue = this.queues[ jobType ] | |
128 | if (queue === undefined) { | |
129 | logger.error('Unknown queue %s to list jobs.', jobType) | |
130 | continue | |
131 | } | |
2c29ad4f | 132 | |
94831479 C |
133 | // FIXME: Bull queue typings does not have getJobs method |
134 | const jobs = await (queue as any).getJobs(state, 0, start + count, asc) | |
135 | results = results.concat(jobs) | |
136 | } | |
94a5ff8a | 137 | |
94831479 C |
138 | results.sort((j1: any, j2: any) => { |
139 | if (j1.timestamp < j2.timestamp) return -1 | |
140 | else if (j1.timestamp === j2.timestamp) return 0 | |
94a5ff8a | 141 | |
94831479 | 142 | return 1 |
94a5ff8a | 143 | }) |
94a5ff8a | 144 | |
94831479 | 145 | if (asc === false) results.reverse() |
94a5ff8a | 146 | |
94831479 | 147 | return results.slice(start, start + count) |
94a5ff8a C |
148 | } |
149 | ||
94831479 C |
150 | async count (state: JobState): Promise<number> { |
151 | let total = 0 | |
3df45638 | 152 | |
94831479 C |
153 | for (const type of jobTypes) { |
154 | const queue = this.queues[ type ] | |
155 | if (queue === undefined) { | |
156 | logger.error('Unknown queue %s to count jobs.', type) | |
157 | continue | |
158 | } | |
3df45638 | 159 | |
94831479 | 160 | const counts = await queue.getJobCounts() |
3df45638 | 161 | |
94831479 C |
162 | total += counts[ state ] |
163 | } | |
3df45638 | 164 | |
94831479 | 165 | return total |
3df45638 C |
166 | } |
167 | ||
94831479 C |
168 | removeOldJobs () { |
169 | for (const key of Object.keys(this.queues)) { | |
170 | const queue = this.queues[key] | |
171 | queue.clean(JOB_COMPLETED_LIFETIME, 'completed') | |
172 | } | |
2c29ad4f C |
173 | } |
174 | ||
6b616860 C |
175 | private addRepeatableJobs () { |
176 | this.queues['videos-views'].add({}, { | |
177 | repeat: REPEAT_JOBS['videos-views'] | |
178 | }) | |
179 | } | |
180 | ||
94a5ff8a C |
181 | static get Instance () { |
182 | return this.instance || (this.instance = new this()) | |
183 | } | |
184 | } | |
185 | ||
186 | // --------------------------------------------------------------------------- | |
187 | ||
188 | export { | |
189 | JobQueue | |
190 | } |