aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/initializers/constants.ts
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2023-04-21 14:55:10 +0200
committerChocobozzz <chocobozzz@cpy.re>2023-05-09 08:57:34 +0200
commit0c9668f77901e7540e2c7045eb0f2974a4842a69 (patch)
tree226d3dd1565b0bb56588897af3b8530e6216e96b /server/initializers/constants.ts
parent6bcb854cdea8688a32240bc5719c7d139806e00b (diff)
downloadPeerTube-0c9668f77901e7540e2c7045eb0f2974a4842a69.tar.gz
PeerTube-0c9668f77901e7540e2c7045eb0f2974a4842a69.tar.zst
PeerTube-0c9668f77901e7540e2c7045eb0f2974a4842a69.zip
Implement remote runner jobs in server
Move ffmpeg functions to @shared
Diffstat (limited to 'server/initializers/constants.ts')
-rw-r--r--server/initializers/constants.ts78
1 files changed, 62 insertions, 16 deletions
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 6cad4eb23..279e77421 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils'
6import { 6import {
7 AbuseState, 7 AbuseState,
8 JobType, 8 JobType,
9 RunnerJobState,
9 UserRegistrationState, 10 UserRegistrationState,
10 VideoChannelSyncState, 11 VideoChannelSyncState,
11 VideoImportState, 12 VideoImportState,
@@ -26,7 +27,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
26 27
27// --------------------------------------------------------------------------- 28// ---------------------------------------------------------------------------
28 29
29const LAST_MIGRATION_VERSION = 760 30const LAST_MIGRATION_VERSION = 765
30 31
31// --------------------------------------------------------------------------- 32// ---------------------------------------------------------------------------
32 33
@@ -81,6 +82,10 @@ const SORTABLE_COLUMNS = {
81 82
82 USER_REGISTRATIONS: [ 'createdAt', 'state' ], 83 USER_REGISTRATIONS: [ 'createdAt', 'state' ],
83 84
85 RUNNERS: [ 'createdAt' ],
86 RUNNER_REGISTRATION_TOKENS: [ 'createdAt' ],
87 RUNNER_JOBS: [ 'updatedAt', 'createdAt', 'priority', 'state', 'progress' ],
88
84 VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ], 89 VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ],
85 90
86 // Don't forget to update peertube-search-index with the same values 91 // Don't forget to update peertube-search-index with the same values
@@ -139,6 +144,8 @@ const REMOTE_SCHEME = {
139 WS: 'wss' 144 WS: 'wss'
140} 145}
141 146
147// ---------------------------------------------------------------------------
148
142const JOB_ATTEMPTS: { [id in JobType]: number } = { 149const JOB_ATTEMPTS: { [id in JobType]: number } = {
143 'activitypub-http-broadcast': 1, 150 'activitypub-http-broadcast': 1,
144 'activitypub-http-broadcast-parallel': 1, 151 'activitypub-http-broadcast-parallel': 1,
@@ -160,6 +167,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
160 'video-channel-import': 1, 167 'video-channel-import': 1,
161 'after-video-channel-import': 1, 168 'after-video-channel-import': 1,
162 'move-to-object-storage': 3, 169 'move-to-object-storage': 3,
170 'transcoding-job-builder': 1,
163 'notify': 1, 171 'notify': 1,
164 'federate-video': 1 172 'federate-video': 1
165} 173}
@@ -183,6 +191,7 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
183 'move-to-object-storage': 1, 191 'move-to-object-storage': 1,
184 'video-channel-import': 1, 192 'video-channel-import': 1,
185 'after-video-channel-import': 1, 193 'after-video-channel-import': 1,
194 'transcoding-job-builder': 1,
186 'notify': 5, 195 'notify': 5,
187 'federate-video': 3 196 'federate-video': 3
188} 197}
@@ -207,6 +216,7 @@ const JOB_TTL: { [id in JobType]: number } = {
207 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours 216 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
208 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours 217 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
209 'after-video-channel-import': 60000 * 5, // 5 minutes 218 'after-video-channel-import': 60000 * 5, // 5 minutes
219 'transcoding-job-builder': 60000, // 1 minute
210 'notify': 60000 * 5, // 5 minutes 220 'notify': 60000 * 5, // 5 minutes
211 'federate-video': 60000 * 5 // 5 minutes 221 'federate-video': 60000 * 5 // 5 minutes
212} 222}
@@ -222,21 +232,6 @@ const JOB_PRIORITY = {
222 TRANSCODING: 100 232 TRANSCODING: 100
223} 233}
224 234
225const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job
226const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
227
228const AP_CLEANER = {
229 CONCURRENCY: 10, // How many requests in parallel we do in activitypub-cleaner job
230 UNAVAILABLE_TRESHOLD: 3, // How many attempts we do before removing an unavailable remote resource
231 PERIOD: parseDurationToMs('1 week') // /!\ Has to be sync with REPEAT_JOBS
232}
233
234const REQUEST_TIMEOUTS = {
235 DEFAULT: 7000, // 7 seconds
236 FILE: 30000, // 30 seconds
237 REDUNDANCY: JOB_TTL['video-redundancy']
238}
239
240const JOB_REMOVAL_OPTIONS = { 235const JOB_REMOVAL_OPTIONS = {
241 COUNT: 10000, // Max jobs to store 236 COUNT: 10000, // Max jobs to store
242 237
@@ -256,7 +251,29 @@ const JOB_REMOVAL_OPTIONS = {
256 251
257const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9) 252const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9)
258 253
254const RUNNER_JOBS = {
255 MAX_FAILURES: 5
256}
257
258// ---------------------------------------------------------------------------
259
260const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job
261const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
262
263const AP_CLEANER = {
264 CONCURRENCY: 10, // How many requests in parallel we do in activitypub-cleaner job
265 UNAVAILABLE_TRESHOLD: 3, // How many attempts we do before removing an unavailable remote resource
266 PERIOD: parseDurationToMs('1 week') // /!\ Has to be sync with REPEAT_JOBS
267}
268
269const REQUEST_TIMEOUTS = {
270 DEFAULT: 7000, // 7 seconds
271 FILE: 30000, // 30 seconds
272 REDUNDANCY: JOB_TTL['video-redundancy']
273}
274
259const SCHEDULER_INTERVALS_MS = { 275const SCHEDULER_INTERVALS_MS = {
276 RUNNER_JOB_WATCH_DOG: Math.min(CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE),
260 ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour 277 ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour
261 REMOVE_OLD_JOBS: 60000 * 60, // 1 hour 278 REMOVE_OLD_JOBS: 60000 * 60, // 1 hour
262 UPDATE_VIDEOS: 60000, // 1 minute 279 UPDATE_VIDEOS: 60000, // 1 minute
@@ -410,6 +427,17 @@ const CONSTRAINTS_FIELDS = {
410 CLIENT_STACK_TRACE: { min: 1, max: 15000 }, // Length 427 CLIENT_STACK_TRACE: { min: 1, max: 15000 }, // Length
411 CLIENT_META: { min: 1, max: 5000 }, // Length 428 CLIENT_META: { min: 1, max: 5000 }, // Length
412 CLIENT_USER_AGENT: { min: 1, max: 200 } // Length 429 CLIENT_USER_AGENT: { min: 1, max: 200 } // Length
430 },
431 RUNNERS: {
432 TOKEN: { min: 1, max: 1000 }, // Length
433 NAME: { min: 1, max: 100 }, // Length
434 DESCRIPTION: { min: 1, max: 1000 } // Length
435 },
436 RUNNER_JOBS: {
437 TOKEN: { min: 1, max: 1000 }, // Length
438 REASON: { min: 1, max: 5000 }, // Length
439 ERROR_MESSAGE: { min: 1, max: 5000 }, // Length
440 PROGRESS: { min: 0, max: 100 } // Value
413 } 441 }
414} 442}
415 443
@@ -540,6 +568,17 @@ const VIDEO_PLAYLIST_TYPES: { [ id in VideoPlaylistType ]: string } = {
540 [VideoPlaylistType.WATCH_LATER]: 'Watch later' 568 [VideoPlaylistType.WATCH_LATER]: 'Watch later'
541} 569}
542 570
571const RUNNER_JOB_STATES: { [ id in RunnerJobState ]: string } = {
572 [RunnerJobState.PROCESSING]: 'Processing',
573 [RunnerJobState.COMPLETED]: 'Completed',
574 [RunnerJobState.PENDING]: 'Pending',
575 [RunnerJobState.ERRORED]: 'Errored',
576 [RunnerJobState.WAITING_FOR_PARENT_JOB]: 'Waiting for parent job to finish',
577 [RunnerJobState.CANCELLED]: 'Cancelled',
578 [RunnerJobState.PARENT_ERRORED]: 'Parent job failed',
579 [RunnerJobState.PARENT_CANCELLED]: 'Parent job cancelled'
580}
581
543const MIMETYPES = { 582const MIMETYPES = {
544 AUDIO: { 583 AUDIO: {
545 MIMETYPE_EXT: { 584 MIMETYPE_EXT: {
@@ -594,6 +633,11 @@ const MIMETYPES = {
594 MIMETYPE_EXT: { 633 MIMETYPE_EXT: {
595 'application/x-bittorrent': '.torrent' 634 'application/x-bittorrent': '.torrent'
596 } 635 }
636 },
637 M3U8: {
638 MIMETYPE_EXT: {
639 'application/vnd.apple.mpegurl': '.m3u8'
640 }
597 } 641 }
598} 642}
599MIMETYPES.AUDIO.EXT_MIMETYPE = invert(MIMETYPES.AUDIO.MIMETYPE_EXT) 643MIMETYPES.AUDIO.EXT_MIMETYPE = invert(MIMETYPES.AUDIO.MIMETYPE_EXT)
@@ -1027,6 +1071,7 @@ export {
1027 SEARCH_INDEX, 1071 SEARCH_INDEX,
1028 DIRECTORIES, 1072 DIRECTORIES,
1029 RESUMABLE_UPLOAD_SESSION_LIFETIME, 1073 RESUMABLE_UPLOAD_SESSION_LIFETIME,
1074 RUNNER_JOB_STATES,
1030 P2P_MEDIA_LOADER_PEER_VERSION, 1075 P2P_MEDIA_LOADER_PEER_VERSION,
1031 ACTOR_IMAGES_SIZE, 1076 ACTOR_IMAGES_SIZE,
1032 ACCEPT_HEADERS, 1077 ACCEPT_HEADERS,
@@ -1085,6 +1130,7 @@ export {
1085 USER_REGISTRATION_STATES, 1130 USER_REGISTRATION_STATES,
1086 LRU_CACHE, 1131 LRU_CACHE,
1087 REQUEST_TIMEOUTS, 1132 REQUEST_TIMEOUTS,
1133 RUNNER_JOBS,
1088 MAX_LOCAL_VIEWER_WATCH_SECTIONS, 1134 MAX_LOCAL_VIEWER_WATCH_SECTIONS,
1089 USER_PASSWORD_RESET_LIFETIME, 1135 USER_PASSWORD_RESET_LIFETIME,
1090 USER_PASSWORD_CREATE_LIFETIME, 1136 USER_PASSWORD_CREATE_LIFETIME,