diff options
-rw-r--r-- | server/initializers/constants.ts | 13 | ||||
-rw-r--r-- | server/lib/job-queue/job-queue.ts | 10 |
2 files changed, 15 insertions, 8 deletions
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index feb45e4d0..5bfeb3746 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -100,9 +100,18 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = { | |||
100 | 'video-import': 1, | 100 | 'video-import': 1, |
101 | 'email': 5 | 101 | 'email': 5 |
102 | } | 102 | } |
103 | const JOB_TTL: { [ id in JobType ]: number } = { | ||
104 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes | ||
105 | 'activitypub-http-unicast': 60000 * 10, // 10 minutes | ||
106 | 'activitypub-http-fetcher': 60000 * 10, // 10 minutes | ||
107 | 'activitypub-follow': 60000 * 10, // 10 minutes | ||
108 | 'video-file-import': 1000 * 3600, // 1 hour | ||
109 | 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long | ||
110 | 'video-import': 1000 * 3600 * 5, // 5 hours | ||
111 | 'email': 60000 * 10 // 10 minutes | ||
112 | } | ||
103 | const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job | 113 | const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job |
104 | const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds | 114 | const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds |
105 | const JOB_REQUEST_TTL = 60000 * 10 // 10 minutes | ||
106 | const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days | 115 | const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days |
107 | 116 | ||
108 | // 1 hour | 117 | // 1 hour |
@@ -576,6 +585,7 @@ export { | |||
576 | ROUTE_CACHE_LIFETIME, | 585 | ROUTE_CACHE_LIFETIME, |
577 | SORTABLE_COLUMNS, | 586 | SORTABLE_COLUMNS, |
578 | FEEDS, | 587 | FEEDS, |
588 | JOB_TTL, | ||
579 | NSFW_POLICY_TYPES, | 589 | NSFW_POLICY_TYPES, |
580 | STATIC_MAX_AGE, | 590 | STATIC_MAX_AGE, |
581 | STATIC_PATHS, | 591 | STATIC_PATHS, |
@@ -592,7 +602,6 @@ export { | |||
592 | VIDEO_TRANSCODING_FPS, | 602 | VIDEO_TRANSCODING_FPS, |
593 | FFMPEG_NICE, | 603 | FFMPEG_NICE, |
594 | JOB_REQUEST_TIMEOUT, | 604 | JOB_REQUEST_TIMEOUT, |
595 | JOB_REQUEST_TTL, | ||
596 | USER_PASSWORD_RESET_LIFETIME, | 605 | USER_PASSWORD_RESET_LIFETIME, |
597 | IMAGE_MIMETYPE_EXT, | 606 | IMAGE_MIMETYPE_EXT, |
598 | SCHEDULER_INTERVALS_MS, | 607 | SCHEDULER_INTERVALS_MS, |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index ffd948b5f..8a24604e1 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -2,7 +2,7 @@ import * as Bull from 'bull' | |||
2 | import { JobState, JobType } from '../../../shared/models' | 2 | import { JobState, JobType } from '../../../shared/models' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { Redis } from '../redis' | 4 | import { Redis } from '../redis' |
5 | import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_REQUEST_TTL } from '../../initializers' | 5 | import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL } from '../../initializers' |
6 | import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' | 6 | import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' |
7 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | 7 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' |
8 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' | 8 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' |
@@ -79,6 +79,7 @@ class JobQueue { | |||
79 | const handler = handlers[handlerName] | 79 | const handler = handlers[handlerName] |
80 | 80 | ||
81 | queue.process(JOB_CONCURRENCY[handlerName], handler) | 81 | queue.process(JOB_CONCURRENCY[handlerName], handler) |
82 | .catch(err => logger.error('Error in job queue processor %s.', handlerName, { err })) | ||
82 | 83 | ||
83 | queue.on('failed', (job, err) => { | 84 | queue.on('failed', (job, err) => { |
84 | logger.error('Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err }) | 85 | logger.error('Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err }) |
@@ -109,11 +110,8 @@ class JobQueue { | |||
109 | 110 | ||
110 | const jobArgs: Bull.JobOptions = { | 111 | const jobArgs: Bull.JobOptions = { |
111 | backoff: { delay: 60 * 1000, type: 'exponential' }, | 112 | backoff: { delay: 60 * 1000, type: 'exponential' }, |
112 | attempts: JOB_ATTEMPTS[obj.type] | 113 | attempts: JOB_ATTEMPTS[obj.type], |
113 | } | 114 | timeout: JOB_TTL[obj.type] |
114 | |||
115 | if (jobsWithRequestTimeout[obj.type] === true) { | ||
116 | jobArgs.timeout = JOB_REQUEST_TTL | ||
117 | } | 115 | } |
118 | 116 | ||
119 | return queue.add(obj.payload, jobArgs) | 117 | return queue.add(obj.payload, jobArgs) |