aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--server/initializers/constants.ts13
-rw-r--r--server/lib/job-queue/job-queue.ts10
2 files changed, 15 insertions, 8 deletions
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index feb45e4d0..5bfeb3746 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -100,9 +100,18 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
100 'video-import': 1, 100 'video-import': 1,
101 'email': 5 101 'email': 5
102} 102}
103const JOB_TTL: { [ id in JobType ]: number } = {
104 'activitypub-http-broadcast': 60000 * 10, // 10 minutes
105 'activitypub-http-unicast': 60000 * 10, // 10 minutes
106 'activitypub-http-fetcher': 60000 * 10, // 10 minutes
107 'activitypub-follow': 60000 * 10, // 10 minutes
108 'video-file-import': 1000 * 3600, // 1 hour
109 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long
110 'video-import': 1000 * 3600 * 5, // 5 hours
111 'email': 60000 * 10 // 10 minutes
112}
103const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job 113const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
104const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds 114const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
105const JOB_REQUEST_TTL = 60000 * 10 // 10 minutes
106const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days 115const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
107 116
108// 1 hour 117// 1 hour
@@ -576,6 +585,7 @@ export {
576 ROUTE_CACHE_LIFETIME, 585 ROUTE_CACHE_LIFETIME,
577 SORTABLE_COLUMNS, 586 SORTABLE_COLUMNS,
578 FEEDS, 587 FEEDS,
588 JOB_TTL,
579 NSFW_POLICY_TYPES, 589 NSFW_POLICY_TYPES,
580 STATIC_MAX_AGE, 590 STATIC_MAX_AGE,
581 STATIC_PATHS, 591 STATIC_PATHS,
@@ -592,7 +602,6 @@ export {
592 VIDEO_TRANSCODING_FPS, 602 VIDEO_TRANSCODING_FPS,
593 FFMPEG_NICE, 603 FFMPEG_NICE,
594 JOB_REQUEST_TIMEOUT, 604 JOB_REQUEST_TIMEOUT,
595 JOB_REQUEST_TTL,
596 USER_PASSWORD_RESET_LIFETIME, 605 USER_PASSWORD_RESET_LIFETIME,
597 IMAGE_MIMETYPE_EXT, 606 IMAGE_MIMETYPE_EXT,
598 SCHEDULER_INTERVALS_MS, 607 SCHEDULER_INTERVALS_MS,
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index ffd948b5f..8a24604e1 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -2,7 +2,7 @@ import * as Bull from 'bull'
2import { JobState, JobType } from '../../../shared/models' 2import { JobState, JobType } from '../../../shared/models'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { Redis } from '../redis' 4import { Redis } from '../redis'
5import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_REQUEST_TTL } from '../../initializers' 5import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL } from '../../initializers'
6import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' 6import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast'
7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' 7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher'
8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' 8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast'
@@ -79,6 +79,7 @@ class JobQueue {
79 const handler = handlers[handlerName] 79 const handler = handlers[handlerName]
80 80
81 queue.process(JOB_CONCURRENCY[handlerName], handler) 81 queue.process(JOB_CONCURRENCY[handlerName], handler)
82 .catch(err => logger.error('Error in job queue processor %s.', handlerName, { err }))
82 83
83 queue.on('failed', (job, err) => { 84 queue.on('failed', (job, err) => {
84 logger.error('Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err }) 85 logger.error('Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err })
@@ -109,11 +110,8 @@ class JobQueue {
109 110
110 const jobArgs: Bull.JobOptions = { 111 const jobArgs: Bull.JobOptions = {
111 backoff: { delay: 60 * 1000, type: 'exponential' }, 112 backoff: { delay: 60 * 1000, type: 'exponential' },
112 attempts: JOB_ATTEMPTS[obj.type] 113 attempts: JOB_ATTEMPTS[obj.type],
113 } 114 timeout: JOB_TTL[obj.type]
114
115 if (jobsWithRequestTimeout[obj.type] === true) {
116 jobArgs.timeout = JOB_REQUEST_TTL
117 } 115 }
118 116
119 return queue.add(obj.payload, jobArgs) 117 return queue.add(obj.payload, jobArgs)