]>
Commit | Line | Data |
---|---|---|
94831479 | 1 | import * as Bull from 'bull' |
628d28e8 | 2 | import { JobState, JobType } from '../../../shared/models' |
94a5ff8a | 3 | import { logger } from '../../helpers/logger' |
71e3dfda | 4 | import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_REQUEST_TTL } from '../../initializers' |
94a5ff8a C |
5 | import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' |
6 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | |
7 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' | |
ecb4e35f | 8 | import { EmailPayload, processEmail } from './handlers/email' |
94831479 | 9 | import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' |
5350fd8e | 10 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' |
94a5ff8a C |
11 | |
12 | type CreateJobArgument = | |
13 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | |
14 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | | |
15 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | | |
5350fd8e | 16 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | |
28be8916 | 17 | { type: 'video-file-import', payload: VideoFileImportPayload } | |
ecb4e35f C |
18 | { type: 'video-file', payload: VideoFilePayload } | |
19 | { type: 'email', payload: EmailPayload } | |
94a5ff8a | 20 | |
94831479 | 21 | const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { |
94a5ff8a C |
22 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, |
23 | 'activitypub-http-unicast': processActivityPubHttpUnicast, | |
24 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | |
5350fd8e | 25 | 'activitypub-follow': processActivityPubFollow, |
28be8916 | 26 | 'video-file-import': processVideoFileImport, |
ecb4e35f C |
27 | 'video-file': processVideoFile, |
28 | 'email': processEmail | |
94a5ff8a C |
29 | } |
30 | ||
94831479 C |
31 | const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = { |
32 | 'activitypub-http-broadcast': true, | |
33 | 'activitypub-http-unicast': true, | |
34 | 'activitypub-http-fetcher': true, | |
35 | 'activitypub-follow': true | |
36 | } | |
37 | ||
38 | const jobTypes: JobType[] = [ | |
39 | 'activitypub-follow', | |
71e3dfda | 40 | 'activitypub-http-broadcast', |
71e3dfda | 41 | 'activitypub-http-fetcher', |
94831479 C |
42 | 'activitypub-http-unicast', |
43 | 'email', | |
44 | 'video-file', | |
45 | 'video-file-import' | |
71e3dfda C |
46 | ] |
47 | ||
94a5ff8a C |
48 | class JobQueue { |
49 | ||
50 | private static instance: JobQueue | |
51 | ||
94831479 | 52 | private queues: { [ id in JobType ]?: Bull.Queue } = {} |
94a5ff8a | 53 | private initialized = false |
2c29ad4f | 54 | private jobRedisPrefix: string |
94a5ff8a C |
55 | |
56 | private constructor () {} | |
57 | ||
3df45638 | 58 | async init () { |
94a5ff8a C |
59 | // Already initialized |
60 | if (this.initialized === true) return | |
61 | this.initialized = true | |
62 | ||
94831479 C |
63 | this.jobRedisPrefix = 'bull-' + CONFIG.WEBSERVER.HOST |
64 | const queueOptions = { | |
2c29ad4f | 65 | prefix: this.jobRedisPrefix, |
94a5ff8a C |
66 | redis: { |
67 | host: CONFIG.REDIS.HOSTNAME, | |
68 | port: CONFIG.REDIS.PORT, | |
30c82f0d RK |
69 | auth: CONFIG.REDIS.AUTH, |
70 | db: CONFIG.REDIS.DB | |
94a5ff8a | 71 | } |
94831479 | 72 | } |
ecb4e35f | 73 | |
94831479 C |
74 | for (const handlerName of Object.keys(handlers)) { |
75 | const queue = new Bull(handlerName, queueOptions) | |
76 | const handler = handlers[handlerName] | |
94a5ff8a | 77 | |
94831479 C |
78 | queue.process(JOB_CONCURRENCY[handlerName], handler) |
79 | .catch(err => logger.error('Cannot execute job queue %s.', handlerName, { err })) | |
3df45638 | 80 | |
94831479 C |
81 | queue.on('error', err => { |
82 | logger.error('Error in job queue %s.', handlerName, { err }) | |
83 | process.exit(-1) | |
94a5ff8a | 84 | }) |
94831479 C |
85 | |
86 | this.queues[handlerName] = queue | |
94a5ff8a C |
87 | } |
88 | } | |
89 | ||
94831479 C |
90 | createJob (obj: CreateJobArgument) { |
91 | const queue = this.queues[obj.type] | |
92 | if (queue === undefined) { | |
93 | logger.error('Unknown queue %s: cannot create job.', obj.type) | |
94 | return | |
95 | } | |
94a5ff8a | 96 | |
94831479 C |
97 | const jobArgs: Bull.JobOptions = { |
98 | backoff: { delay: 60 * 1000, type: 'exponential' }, | |
99 | attempts: JOB_ATTEMPTS[obj.type] | |
100 | } | |
71e3dfda | 101 | |
94831479 C |
102 | if (jobsWithRequestTimeout[obj.type] === true) { |
103 | jobArgs.timeout = JOB_REQUEST_TTL | |
104 | } | |
71e3dfda | 105 | |
94831479 | 106 | return queue.add(obj.payload, jobArgs) |
94a5ff8a C |
107 | } |
108 | ||
94831479 C |
109 | async listForApi (state: JobState, start: number, count: number, asc?: boolean): Promise<Bull.Job[]> { |
110 | let results: Bull.Job[] = [] | |
94a5ff8a | 111 | |
94831479 C |
112 | // TODO: optimize |
113 | for (const jobType of jobTypes) { | |
114 | const queue = this.queues[ jobType ] | |
115 | if (queue === undefined) { | |
116 | logger.error('Unknown queue %s to list jobs.', jobType) | |
117 | continue | |
118 | } | |
2c29ad4f | 119 | |
94831479 C |
120 | // FIXME: Bull queue typings does not have getJobs method |
121 | const jobs = await (queue as any).getJobs(state, 0, start + count, asc) | |
122 | results = results.concat(jobs) | |
123 | } | |
94a5ff8a | 124 | |
94831479 C |
125 | results.sort((j1: any, j2: any) => { |
126 | if (j1.timestamp < j2.timestamp) return -1 | |
127 | else if (j1.timestamp === j2.timestamp) return 0 | |
94a5ff8a | 128 | |
94831479 | 129 | return 1 |
94a5ff8a | 130 | }) |
94a5ff8a | 131 | |
94831479 | 132 | if (asc === false) results.reverse() |
94a5ff8a | 133 | |
94831479 | 134 | return results.slice(start, start + count) |
94a5ff8a C |
135 | } |
136 | ||
94831479 C |
137 | async count (state: JobState): Promise<number> { |
138 | let total = 0 | |
3df45638 | 139 | |
94831479 C |
140 | for (const type of jobTypes) { |
141 | const queue = this.queues[ type ] | |
142 | if (queue === undefined) { | |
143 | logger.error('Unknown queue %s to count jobs.', type) | |
144 | continue | |
145 | } | |
3df45638 | 146 | |
94831479 | 147 | const counts = await queue.getJobCounts() |
3df45638 | 148 | |
94831479 C |
149 | total += counts[ state ] |
150 | } | |
3df45638 | 151 | |
94831479 | 152 | return total |
3df45638 C |
153 | } |
154 | ||
94831479 C |
155 | removeOldJobs () { |
156 | for (const key of Object.keys(this.queues)) { | |
157 | const queue = this.queues[key] | |
158 | queue.clean(JOB_COMPLETED_LIFETIME, 'completed') | |
159 | } | |
2c29ad4f C |
160 | } |
161 | ||
94a5ff8a C |
162 | static get Instance () { |
163 | return this.instance || (this.instance = new this()) | |
164 | } | |
165 | } | |
166 | ||
167 | // --------------------------------------------------------------------------- | |
168 | ||
169 | export { | |
170 | JobQueue | |
171 | } |